first commit

src/config/celery.py

@@ -0,0 +1,42 @@
+"""
+Celery configuration for the project.
+
+This module contains Celery configuration and task registration.
+"""
+
+import os
+
+from celery import Celery
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.development")
+
+app = Celery("project")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
+
+# Configure Celery Beat schedule
+app.conf.beat_schedule = {
+    "check-pending-scraping-jobs": {
+        "task": "apps.scraping.tasks.check_pending_jobs",
+        "schedule": 300.0,  # Every 5 minutes
+    },
+    "process-extracted-data": {
+        "task": "apps.data_processor.tasks.process_extracted_data",
+        "schedule": 600.0,  # Every 10 minutes
+    },
+}
+
+app.conf.timezone = "UTC"
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print(f"Request: {self.request!r}")