--- a/src/pyams_file/script/__init__.py Wed Feb 06 09:45:52 2019 +0100
+++ b/src/pyams_file/script/__init__.py Mon Feb 11 14:04:52 2019 +0100
@@ -12,35 +12,76 @@
__docformat__ = 'restructuredtext'
-from ZODB.utils import oid_repr, p64
+import os
+
+from ZODB.POSException import POSKeyError
+from ZODB.utils import oid_repr, p64, repr_to_oid
+from zope.intid.interfaces import IIntIds
from zope.lifecycleevent import ObjectRemovedEvent
from pyams_file.interfaces import IBlobReferenceManager
+from pyams_utils.decorator import deprecated
from pyams_utils.registry import get_global_registry, get_utility, set_local_registry
-def remove_unused_blobs(root):
- """Blobs removal scripts
+def check_before_upgrade(root):
+ """Register all missing files before upgrading
- If you used previous versions of PyAMS_file package, files and blobs references where not handled
- correctly and this file is intended to be used from *pshell* after running an *mdtools.relstorage*
- command like this:
+ Run a ZODB search before upgrading to last PyAMS_file package:
/var/local/env/pyams$ ./bin/zodbsearch --config etc/zodb-relstorage.conf \
--data pyams_file.file.File pyams_file.file.ImageFile pyams_file.file.SVGImageFile \
pyams_file.file.VideoFile pyams_file.file.AudioFile 2>&1 \
| grep found | awk '{print $7}' | sort -u | cut -d \" -f 2 > var/FILES.oids
- P.S.: this command can be used with any database storage (ZEO, Relstorage or Newt)!!!
+
+ """
+ set_local_registry(root.getSiteManager())
+
+ db = getattr(root, '_p_jar')
+ intids = get_utility(IIntIds)
- This command will generate a FILES.oids file in "var" directory, containing the OIDs of all
- files stored in the ZODB.
+ count = 0
+ files = open('var/FILES.oids', 'r')
+ for oid in files.readlines():
+ file_oid = p64(int(oid, 16))
+ file = db.get(file_oid)
+ ref = intids.queryId(file)
+ if ref is None:
+ intids.register(file)
+ count += 1
+ return count
+
+
+def remove_unused_fs_blobs(root):
+ """Filesystem blobs removal script
+
+ Find all blobs before running this command, and store their path to file:
- The script will then read this file and get access to all stored files; if the file blob is not referenced
- into blobs references manager, it can be removed. Removal is done by removing all references to it, so that
- the blob file will be physically removed after next database pack.
+ /var/local/env/pyams$ find /var/db/blobs -name "*.blob" | cut -d \/ -f 4- > var/BLOBS.fs
+
"""
+ db = getattr(root, '_p_jar')
+ blobs = open('var/BLOBS.fs', 'r')
+ for filename in blobs.readlines():
+ oid, tid = filename.rsplit('/', 1)
+ blob_oid = repr_to_oid(oid.replace('/0x', '').encode())
+ try:
+ db.get(blob_oid)
+ except POSKeyError:
+ print("Removing file: {}".format(filename[:-1]))
+ try:
+ os.remove('var/db/blobs/{}'.format(filename[:-1]))
+ os.remove('var/db/blobs/{}/.lock'.format(oid))
+ os.removedirs('var/db/blobs/{}'.format(oid))
+ except OSError:
+ pass
+
+
+@deprecated
+def remove_unused_blobs(root):
+ """This method is deprecated and may generate errors: DON'T USE IT ANYMORE"""
registry = get_global_registry()
set_local_registry(root.getSiteManager())
references = get_utility(IBlobReferenceManager)