# HG changeset patch
# User Thierry Florac <thierry.florac@onf.fr>
# Date 1549890292 -3600
# Node ID e269569348c66af222b75f63cc358882891ef706
# Parent  df9ed2ac2641ca5fa51bd0a060900e6bd6e3397d
Updated helper scripts

diff -r df9ed2ac2641 -r e269569348c6 src/pyams_file/script/__init__.py
--- a/src/pyams_file/script/__init__.py	Wed Feb 06 09:45:52 2019 +0100
+++ b/src/pyams_file/script/__init__.py	Mon Feb 11 14:04:52 2019 +0100
@@ -12,35 +12,76 @@
 
 __docformat__ = 'restructuredtext'
 
-from ZODB.utils import oid_repr, p64
+import os
+
+from ZODB.POSException import POSKeyError
+from ZODB.utils import oid_repr, p64, repr_to_oid
+from zope.intid.interfaces import IIntIds
 from zope.lifecycleevent import ObjectRemovedEvent
 
 from pyams_file.interfaces import IBlobReferenceManager
+from pyams_utils.decorator import deprecated
 from pyams_utils.registry import get_global_registry, get_utility, set_local_registry
 
 
-def remove_unused_blobs(root):
-    """Blobs removal scripts
+def check_before_upgrade(root):
+    """Register all missing files before upgrading
 
-    If you used previous versions of PyAMS_file package, files and blobs references where not handled
-    correctly and this file is intended to be used from *pshell* after running an *mdtools.relstorage*
-    command like this:
+    Run a ZODB search before upgrading to last PyAMS_file package:
 
     /var/local/env/pyams$ ./bin/zodbsearch --config etc/zodb-relstorage.conf \
         --data pyams_file.file.File pyams_file.file.ImageFile pyams_file.file.SVGImageFile \
         pyams_file.file.VideoFile pyams_file.file.AudioFile 2>&1 \
         | grep found | awk '{print $7}' | sort -u | cut -d \" -f 2 > var/FILES.oids
 
-    P.S.: this command can be used with any database storage (ZEO, Relstorage or Newt)!!!
+
+    """
+    set_local_registry(root.getSiteManager())
+
+    db = getattr(root, '_p_jar')
+    intids = get_utility(IIntIds)
 
-    This command will generate a FILES.oids file in "var" directory, containing the OIDs of all
-    files stored in the ZODB.
+    count = 0
+    files = open('var/FILES.oids', 'r')
+    for oid in files.readlines():
+        file_oid = p64(int(oid, 16))
+        file = db.get(file_oid)
+        ref = intids.queryId(file)
+        if ref is None:
+            intids.register(file)
+            count += 1
+    return count
+
+
+def remove_unused_fs_blobs(root):
+    """Filesystem blobs removal script
+
+    Find all blobs before running this command, and store their path to file:
 
-    The script will then read this file and get access to all stored files; if the file blob is not referenced
-    into blobs references manager, it can be removed. Removal is done by removing all references to it, so that
-    the blob file will be physically removed after next database pack.
+        /var/local/env/pyams$ find /var/db/blobs -name "*.blob" | cut -d \/ -f 4- > var/BLOBS.fs
+
     """
+    db = getattr(root, '_p_jar')
+    blobs = open('var/BLOBS.fs', 'r')
 
+    for filename in blobs.readlines():
+        oid, tid = filename.rsplit('/', 1)
+        blob_oid = repr_to_oid(oid.replace('/0x', '').encode())
+        try:
+            db.get(blob_oid)
+        except POSKeyError:
+            print("Removing file: {}".format(filename[:-1]))
+            try:
+                os.remove('var/db/blobs/{}'.format(filename[:-1]))
+                os.remove('var/db/blobs/{}/.lock'.format(oid))
+                os.removedirs('var/db/blobs/{}'.format(oid))
+            except OSError:
+                pass
+
+
+@deprecated
+def remove_unused_blobs(root):
+    """This method is deprecated and may generate errors: DON'T USE IT ANYMORE"""
     registry = get_global_registry()
     set_local_registry(root.getSiteManager())
     references = get_utility(IBlobReferenceManager)