--- a/src/pyams_file/file.py	Tue Feb 05 14:48:54 2019 +0100
+++ b/src/pyams_file/file.py	Tue Feb 05 15:07:06 2019 +0100
@@ -141,13 +141,16 @@
 
     def drop_reference(self, blob, reference):
         oid = oid_repr(getattr(blob, '_p_oid'))
-        refs = self.refs.get(oid) or set()
-        if reference in refs:
-            refs.remove(reference)
-        if refs:
-            self.refs[oid] = refs
+        refs = self.refs.get(oid)
+        if refs is not None:
+            if reference in refs:
+                refs.remove(reference)
+            if refs:
+                self.refs[oid] = refs
+            else:
+                del self.refs[oid]
+                del blob
         else:
-            del self.refs[oid]
             del blob