okay fine

This commit is contained in:
pacnpal
2024-11-03 17:47:26 +00:00
parent 01c6004a79
commit 27eb239e97
10020 changed files with 1935769 additions and 2364 deletions

View File

@@ -0,0 +1,7 @@
'''
django-cleanup automatically deletes files for FileField, ImageField, and
subclasses. It will delete old files when a new file is being save and it
will delete files on model instance deletion.
'''
__version__ = '9.0.0'

View File

@@ -0,0 +1,24 @@
'''
AppConfig for django-cleanup, prepare the cache and connect signal handlers
'''
from django.apps import AppConfig
from . import cache, handlers
class CleanupConfig(AppConfig):
name = 'django_cleanup'
verbose_name = 'Django Cleanup'
default = True
def ready(self):
cache.prepare(False)
handlers.connect()
class CleanupSelectedConfig(AppConfig):
name = 'django_cleanup'
verbose_name = 'Django Cleanup'
def ready(self):
cache.prepare(True)
handlers.connect()

View File

@@ -0,0 +1,185 @@
''' Our local cache of filefields, everything is private to this package.'''
from collections import defaultdict
from django.apps import apps
from django.db import models
from django.utils.module_loading import import_string
CACHE_NAME = '_django_cleanup_original_cache'
def fields_default():
return set()
FIELDS = defaultdict(fields_default)
def fields_dict_default():
return {}
FIELDS_FIELDS = defaultdict(fields_dict_default)
FIELDS_STORAGE = defaultdict(fields_dict_default)
# cache init ##
def prepare(select_mode):
'''Prepare the cache for all models, non-reentrant'''
if FIELDS: # pragma: no cover
return
for model in apps.get_models():
if ignore_model(model, select_mode):
continue
name = get_model_name(model)
if model_has_filefields(name): # pragma: no cover
continue
opts = model._meta
for field in opts.get_fields():
if isinstance(field, models.FileField):
add_field_for_model(name, field.name, field)
def add_field_for_model(model_name, field_name, field):
'''Centralized function to make all our local caches.'''
# store models that have filefields and the field names
FIELDS[model_name].add(field_name)
# store the dotted path of the field class for each field
# in case we need to restore it later on
FIELDS_FIELDS[model_name][field_name] = get_dotted_path(field)
# also store the dotted path of the storage for the same reason
FIELDS_STORAGE[model_name][field_name] = get_dotted_path(field.storage)
# generators ##
def get_fields_for_model(model_name, exclude=None):
'''Get the filefields for a model if it has them'''
if model_has_filefields(model_name):
fields = FIELDS[model_name]
if exclude is not None:
assert isinstance(exclude, set)
fields = fields.difference(exclude)
for field_name in fields:
yield field_name
def fields_for_model_instance(instance, using=None):
'''
Yields (name, descriptor) for each file field given an instance
Can use the `using` kwarg to change the instance that the `FieldFile`
will receive.
'''
if using is None:
using = instance
model_name = get_model_name(instance)
deferred_fields = instance.get_deferred_fields()
for field_name in get_fields_for_model(model_name, exclude=deferred_fields):
fieldfile = getattr(instance, field_name)
yield field_name, fieldfile.__class__(using, fieldfile.field, fieldfile.name)
# restore ##
def get_field(model_name, field_name):
'''Restore a field from its dotted path'''
return import_string(FIELDS_FIELDS[model_name][field_name])
def get_field_storage(model_name, field_name):
'''Restore a storage from its dotted path'''
return import_string(FIELDS_STORAGE[model_name][field_name])
# utilities ##
def get_dotted_path(object_):
'''get the dotted path for an object'''
klass = object_.__class__
return f'{klass.__module__}.{klass.__qualname__}'
def get_model_name(model):
'''returns a unique model name'''
opt = model._meta
return f'{opt.app_label}.{opt.model_name}'
def get_mangled_ignore(model):
'''returns a mangled attribute name specific to the model for ignore functionality'''
opt = model._meta
return f'_{opt.model_name}__{opt.app_label}_cleanup_ignore'
def get_mangled_select(model):
'''returns a mangled attribute name specific to the model for select functionality'''
opt = model._meta
return f'_{opt.model_name}__{opt.app_label}_cleanup_select'
# booleans ##
def model_has_filefields(model_name):
'''Check if a model has filefields'''
return model_name in FIELDS
def ignore_model(model, select_mode):
'''Check if a model should be ignored'''
return ((not hasattr(model, get_mangled_select(model)))
if select_mode else hasattr(model, get_mangled_ignore(model)))
# instance functions ##
def remove_instance_cache(instance):
'''Remove the cache from an instance'''
if has_cache(instance):
delattr(instance, CACHE_NAME)
def make_cleanup_cache(instance, source=None):
'''
Make the cleanup cache for an instance.
Can also change the source of the data with the `source` kwarg.
'''
if source is None:
source = instance
setattr(instance, CACHE_NAME, dict(
fields_for_model_instance(source, using=instance)))
def has_cache(instance):
'''Check if an instance has a cache on it'''
return hasattr(instance, CACHE_NAME)
def get_field_attr(instance, field_name):
'''Get a value from the cache on an instance'''
return getattr(instance, CACHE_NAME)[field_name]
# data sharing ##
def cleanup_models():
'''Get all the models we have in the FIELDS cache'''
for model_name in FIELDS:
yield apps.get_model(model_name)
def cleanup_fields():
'''Get a copy of the FIELDS cache'''
return FIELDS.copy()

View File

@@ -0,0 +1,26 @@
'''Public utilities'''
from .cache import (
get_mangled_ignore as _get_mangled_ignore, get_mangled_select as _get_mangled_select,
make_cleanup_cache as _make_cleanup_cache)
__all__ = ['refresh', 'cleanup_ignore', 'cleanup_select']
def refresh(instance):
'''Refresh the cache for an instance'''
return _make_cleanup_cache(instance)
def ignore(cls):
'''Mark a model to ignore for cleanup'''
setattr(cls, _get_mangled_ignore(cls), None)
return cls
cleanup_ignore = ignore
def select(cls):
'''Mark a model to select for cleanup'''
setattr(cls, _get_mangled_select(cls), None)
return cls
cleanup_select = select

View File

@@ -0,0 +1,134 @@
'''
Signal handlers to manage FileField files.
'''
import logging
from django.db.models.signals import post_delete, post_init, post_save, pre_save
from django.db.transaction import on_commit
from . import cache
from .signals import cleanup_post_delete, cleanup_pre_delete
logger = logging.getLogger(__name__)
class FakeInstance:
'''A Fake model instance to ensure an instance is not modified'''
def cache_original_post_init(sender, instance, **kwargs):
'''Post_init on all models with file fields, saves original values'''
cache.make_cleanup_cache(instance)
def fallback_pre_save(sender, instance, raw, update_fields, using, **kwargs):
'''Fallback to the database to remake the cleanup cache if there is none'''
if raw: # pragma: no cover
return
if instance.pk and not cache.has_cache(instance):
try:
db_instance = sender.objects.get(pk=instance.pk)
except sender.DoesNotExist: # pragma: no cover
return
cache.make_cleanup_cache(instance, source=db_instance)
def delete_old_post_save(sender, instance, raw, created, update_fields, using,
**kwargs):
'''Post_save on all models with file fields, deletes old files'''
if raw:
return
if not created:
for field_name, new_file in cache.fields_for_model_instance(instance):
if update_fields is None or field_name in update_fields:
old_file = cache.get_field_attr(instance, field_name)
if old_file != new_file:
delete_file(sender, instance, field_name, old_file, using, 'updated')
# reset cache
cache.make_cleanup_cache(instance)
def delete_all_post_delete(sender, instance, using, **kwargs):
'''Post_delete on all models with file fields, deletes all files'''
for field_name, file_ in cache.fields_for_model_instance(instance):
delete_file(sender, instance, field_name, file_, using, 'deleted')
def delete_file(sender, instance, field_name, file_, using, reason):
'''Deletes a file'''
if not file_.name:
return
# add a fake instance to the file being deleted to avoid
# any changes to the real instance.
file_.instance = FakeInstance()
# pickled filefields lose lots of data, and contrary to how it is
# documented, the file descriptor does not recover them
model_name = cache.get_model_name(instance)
# recover the 'field' if necessary
if not hasattr(file_, 'field'):
file_.field = cache.get_field(model_name, field_name)()
file_.field.name = field_name
# if our file name is default don't delete
default = file_.field.default if not callable(file_.field.default) else file_.field.default()
if file_.name == default:
return
# recover the 'storage' if necessary
if not hasattr(file_, 'storage'):
file_.storage = cache.get_field_storage(model_name, field_name)()
event = {
'deleted': reason == 'deleted',
'model_name': model_name,
'field_name': field_name,
'file_name': file_.name,
'default_file_name': default,
'file': file_,
'instance': instance,
'updated': reason == 'updated'
}
# this will run after a successful commit
# assuming you are in a transaction and on a database that supports
# transactions, otherwise it will run immediately
def run_on_commit():
cleanup_pre_delete.send(sender=sender, **event)
success = False
error = None
try:
file_.delete(save=False)
success = True
except Exception as ex:
error = ex
opts = instance._meta
logger.exception(
'There was an exception deleting the file `%s` on field `%s.%s.%s`',
file_, opts.app_label, opts.model_name, field_name)
cleanup_post_delete.send(sender=sender, error=error, success=success, **event)
on_commit(run_on_commit, using)
def connect():
'''Connect signals to the cleanup models'''
for model in cache.cleanup_models():
suffix = f'_django_cleanup_{cache.get_model_name(model)}'
post_init.connect(cache_original_post_init, sender=model,
dispatch_uid=f'post_init{suffix}')
pre_save.connect(fallback_pre_save, sender=model,
dispatch_uid=f'pre_save{suffix}')
post_save.connect(delete_old_post_save, sender=model,
dispatch_uid=f'post_save{suffix}')
post_delete.connect(delete_all_post_delete, sender=model,
dispatch_uid=f'post_delete{suffix}')

View File

@@ -0,0 +1,15 @@
'''
django-cleanup sends the following signals
'''
from django.dispatch import Signal
__all__ = ['cleanup_pre_delete', 'cleanup_post_delete']
cleanup_pre_delete = Signal()
'''Called just before a file is deleted. Passes a `file` keyword argument.'''
cleanup_post_delete = Signal()
'''Called just after a file is deleted. Passes a `file` keyword argument.'''