diff --git a/dojo/db_migrations/0276_finding_lifecycle_event.py b/dojo/db_migrations/0276_finding_lifecycle_event.py new file mode 100644 index 00000000000..b39346423a2 --- /dev/null +++ b/dojo/db_migrations/0276_finding_lifecycle_event.py @@ -0,0 +1,29 @@ +# Generated by Django 5.2.14 on 2026-07-04 03:26 + +import django.db.models.deletion +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0275_usercontactinfo_user_state_details'), + ] + + operations = [ + migrations.CreateModel( + name='Finding_Lifecycle_Event', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('actor_type', models.CharField(choices=[('import', 'Import pipeline'), ('dedupe', 'Deduplication'), ('jira', 'JIRA sync'), ('system', 'System')], default='system', editable=False, help_text='Which part of the platform produced this event.', max_length=20, verbose_name='Actor Type')), + ('action', models.CharField(choices=[('created', 'Created'), ('closed', 'Closed'), ('reopened', 'Reopened'), ('marked_duplicate', 'Marked duplicate'), ('pushed_jira', 'Pushed to JIRA')], editable=False, help_text='The lifecycle transition that happened to the finding.', max_length=20, verbose_name='Action')), + ('detail', models.JSONField(blank=True, default=dict, editable=False, help_text='Context for the transition: import/test ids, close reason, duplicate original, JIRA key.', verbose_name='Detail')), + ('created', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='Created')), + ('finding', models.ForeignKey(db_constraint=False, editable=False, on_delete=django.db.models.deletion.DO_NOTHING, related_name='lifecycle_events', to='dojo.finding', verbose_name='Finding')), + ], + options={ + 'indexes': [models.Index(fields=['finding', 'created'], name='finding_lifecycle_f_created'), models.Index(fields=['created'], name='finding_lifecycle_created')], + }, + ), + ] diff --git a/dojo/finding/api/serializer.py b/dojo/finding/api/serializer.py index 360f08b7926..45065bcc1d0 100644 --- a/dojo/finding/api/serializer.py +++ b/dojo/finding/api/serializer.py @@ -33,6 +33,7 @@ Engagement, Finding, Finding_Group, + Finding_Lifecycle_Event, Finding_Template, Note_Type, Product, @@ -148,6 +149,12 @@ def to_representation(self, value): return value +class FindingLifecycleEventSerializer(serializers.ModelSerializer): + class Meta: + model = Finding_Lifecycle_Event + fields = ("id", "action", "actor_type", "detail", "created") + + class BurpRawRequestResponseSerializer(serializers.Serializer): req_resp = RequestResponseSerializerField(required=True) diff --git a/dojo/finding/api/views.py b/dojo/finding/api/views.py index 2df4cf00ad3..4696105bfe0 100644 --- a/dojo/finding/api/views.py +++ b/dojo/finding/api/views.py @@ -40,6 +40,7 @@ BurpRawRequestResponseSerializer, FindingCloseSerializer, FindingCreateSerializer, + FindingLifecycleEventSerializer, FindingMetaSerializer, FindingNoteSerializer, FindingSerializer, @@ -60,6 +61,7 @@ DojoMeta, FileUpload, Finding, + Finding_Lifecycle_Event, Finding_Template, NoteHistory, Notes, @@ -293,6 +295,25 @@ def verify(self, request, pk=None): request=api_v2_serializers.TagSerializer, responses={status.HTTP_201_CREATED: api_v2_serializers.TagSerializer}, ) + @extend_schema( + methods=["GET"], + responses={ + status.HTTP_200_OK: FindingLifecycleEventSerializer(many=True), + }, + ) + @action(detail=True, methods=["get"], url_path="lifecycle_events", permission_classes=(IsAuthenticated, permissions.UserHasFindingRelatedObjectPermission)) + def lifecycle_events(self, request, pk=None): + """ + The finding's provenance timeline: created by which import, closed why, + marked duplicate of what, pushed to JIRA as which key. + """ + finding = self.get_object() + events = ( + Finding_Lifecycle_Event.objects.filter(finding_id=finding.id) + .order_by("-created", "-id")[:500] + ) + return Response(FindingLifecycleEventSerializer(events, many=True).data) + @action(detail=True, methods=["get", "post"], permission_classes=(IsAuthenticated, permissions.UserHasFindingRelatedObjectPermission)) def tags(self, request, pk=None): finding = self.get_object() diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index bf91b610b6d..ec98ac43037 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -8,7 +8,8 @@ from django.db.models.query_utils import Q from dojo.celery import app -from dojo.models import Endpoint_Status, Finding, System_Settings +from dojo.finding.lifecycle import record_lifecycle_event +from dojo.models import Endpoint_Status, Finding, Finding_Lifecycle_Event, System_Settings logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") @@ -192,6 +193,16 @@ def set_duplicate(new_finding, existing_finding, *, save=True): logger.debug("saving existing finding: %d", existing_finding.id) super(Finding, existing_finding).save(skip_validation=True) + # Provenance: record the dedupe decision with enough context to answer + # "why is this a duplicate?" (transitively re-pointed findings record + # their own event through the recursive call above) + record_lifecycle_event( + new_finding.id, + Finding_Lifecycle_Event.Action.MARKED_DUPLICATE, + {"original_id": existing_finding.id, "hash_code": new_finding.hash_code}, + actor_type=Finding_Lifecycle_Event.ActorType.DEDUPE, + ) + return all_modified diff --git a/dojo/finding/lifecycle.py b/dojo/finding/lifecycle.py new file mode 100644 index 00000000000..ac08bfd45ab --- /dev/null +++ b/dojo/finding/lifecycle.py @@ -0,0 +1,86 @@ +""" +Finding lifecycle provenance. + +Records SEMANTIC events on findings — created by import X, closed because it +was gone from a re-upload, marked duplicate of Y, pushed to JIRA as KEY — +the "why", which neither field-level history (pghistory triggers) nor the +per-import action records (Test_Import_Finding_Action) can express. + +Write discipline (this table must never become a performance problem): +- Transition-only: no rows for "matched, nothing changed" reimports. +- Batched: importers collect events and bulk_create them per finding batch. +- No signals, no per-row saves; detail values are truncated. +- The FK carries no database constraint and on_delete=DO_NOTHING, so bulk + finding deletion never walks this table; orphans are swept by retention. +""" +import logging +from datetime import timedelta + +from django.conf import settings +from django.utils import timezone + +from dojo.celery import app +from dojo.models import Finding_Lifecycle_Event + +logger = logging.getLogger(__name__) + +_DETAIL_MAX_CHARS = 256 +_PURGE_BATCH_SIZE = 10000 + + +def lifecycle_events_enabled() -> bool: + return getattr(settings, "FINDING_LIFECYCLE_EVENTS_ENABLED", True) + + +def _truncate(value): + if isinstance(value, str) and len(value) > _DETAIL_MAX_CHARS: + return value[: _DETAIL_MAX_CHARS - 1] + "…" + return value + + +def lifecycle_event(finding_id, action, detail=None, actor_type=Finding_Lifecycle_Event.ActorType.IMPORT): + """Build an unsaved event; persist via record_lifecycle_events.""" + detail = {k: _truncate(v) for k, v in (detail or {}).items() if v is not None} + return Finding_Lifecycle_Event( + finding_id=finding_id, + actor_type=actor_type, + action=action, + detail=detail, + ) + + +def record_lifecycle_events(events) -> None: + """Bulk-persist events. Cheap no-op when disabled or empty.""" + if not events or not lifecycle_events_enabled(): + return + Finding_Lifecycle_Event.objects.bulk_create(events, batch_size=1000) + + +def record_lifecycle_event(finding_id, action, detail=None, actor_type=Finding_Lifecycle_Event.ActorType.IMPORT) -> None: + if not lifecycle_events_enabled(): + return + record_lifecycle_events([lifecycle_event(finding_id, action, detail, actor_type)]) + + +@app.task +def purge_finding_lifecycle_events(*args, **kwargs): + """ + Delete lifecycle events older than the retention window, in batches. + Also sweeps events orphaned by finding deletion (the FK intentionally + carries no constraint so deletes never pay for this table). + """ + retention_days = getattr(settings, "FINDING_LIFECYCLE_EVENTS_RETENTION_DAYS", 540) + cutoff = timezone.now() - timedelta(days=retention_days) + total = 0 + while True: + batch_ids = list( + Finding_Lifecycle_Event.objects.filter(created__lt=cutoff) + .values_list("id", flat=True)[:_PURGE_BATCH_SIZE], + ) + if not batch_ids: + break + deleted, _ = Finding_Lifecycle_Event.objects.filter(id__in=batch_ids).delete() + total += deleted + if total: + logger.info("purged %d finding lifecycle events older than %d days", total, retention_days) + return total diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 8a55d3df9e6..3d22a0b3e9a 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -1555,6 +1555,77 @@ class CWE(models.Model): number = models.IntegerField() +class Finding_Lifecycle_Event(models.Model): + + """ + Append-only provenance log of SEMANTIC finding transitions: created by + import X, closed because gone from a re-upload, marked duplicate of Y, + pushed to JIRA as KEY. Complements (does not duplicate) field-level + history and Test_Import_Finding_Action — this table records the WHY. + + Deliberately skinny and delete-safe: the finding FK carries no database + constraint and on_delete=DO_NOTHING, so bulk finding deletion never + touches this table; orphans are swept by the retention purge task. + Writes are transition-only and batched (see dojo/finding/lifecycle.py). + """ + + class ActorType(models.TextChoices): + IMPORT = "import", _("Import pipeline") + DEDUPE = "dedupe", _("Deduplication") + JIRA = "jira", _("JIRA sync") + SYSTEM = "system", _("System") + + class Action(models.TextChoices): + CREATED = "created", _("Created") + CLOSED = "closed", _("Closed") + REOPENED = "reopened", _("Reopened") + MARKED_DUPLICATE = "marked_duplicate", _("Marked duplicate") + PUSHED_JIRA = "pushed_jira", _("Pushed to JIRA") + + finding = models.ForeignKey( + "dojo.Finding", + on_delete=models.DO_NOTHING, + db_constraint=False, + related_name="lifecycle_events", + editable=False, + verbose_name=_("Finding"), + ) + actor_type = models.CharField( + max_length=20, + choices=ActorType.choices, + default=ActorType.SYSTEM, + editable=False, + verbose_name=_("Actor Type"), + help_text=_("Which part of the platform produced this event."), + ) + action = models.CharField( + max_length=20, + choices=Action.choices, + editable=False, + verbose_name=_("Action"), + help_text=_("The lifecycle transition that happened to the finding."), + ) + detail = models.JSONField( + default=dict, + blank=True, + editable=False, + verbose_name=_("Detail"), + help_text=_("Context for the transition: import/test ids, close reason, duplicate original, JIRA key."), + ) + created = models.DateTimeField(default=timezone.now, editable=False, verbose_name=_("Created")) + + class Meta: + indexes = [ + # the only hot read: one finding's timeline, newest first + models.Index(fields=["finding", "created"], name="finding_lifecycle_f_created"), + # retention purge scans by age + models.Index(fields=["created"], name="finding_lifecycle_created"), + ] + + def __str__(self): + return f"{self.action} finding {self.finding_id} ({self.actor_type})" + + class BurpRawRequestResponse(models.Model): finding = models.ForeignKey("dojo.Finding", blank=True, null=True, on_delete=models.CASCADE) burpRequestBase64 = models.BinaryField() diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index fe015b610b0..c6e421da290 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -12,6 +12,7 @@ import dojo.finding.helper as finding_helper import dojo.risk_acceptance.helper as ra_helper +from dojo.finding.lifecycle import record_lifecycle_event from dojo.importers.options import ImporterOptions from dojo.jira.services import is_keep_in_sync from dojo.location.models import Location @@ -27,6 +28,7 @@ Endpoint, FileUpload, Finding, + Finding_Lifecycle_Event, Test, Test_Import, Test_Import_Finding_Action, @@ -860,6 +862,12 @@ def mitigate_finding( finding.save(dedupe_option=False, product_grading_option=product_grading_option) else: finding.save(dedupe_option=False, push_to_jira=(self.push_to_jira or is_keep_in_sync(finding, prefetched_jira_instance=self.jira_instance)), product_grading_option=product_grading_option) + # Provenance: record WHY the finding closed (close_old_findings / re-upload) + record_lifecycle_event( + finding.id, + Finding_Lifecycle_Event.Action.CLOSED, + {"test_id": self.test.id, "scan_type": self.scan_type, "reason": note_message}, + ) def notify_scan_added( self, diff --git a/dojo/importers/default_importer.py b/dojo/importers/default_importer.py index 6cdcb699024..a631428825d 100644 --- a/dojo/importers/default_importer.py +++ b/dojo/importers/default_importer.py @@ -7,6 +7,7 @@ from dojo.celery_dispatch import dojo_dispatch_task from dojo.finding import helper as finding_helper +from dojo.finding.lifecycle import lifecycle_event, record_lifecycle_events from dojo.importers.base_importer import BaseImporter, Parser from dojo.importers.base_location_manager import LocationHandler from dojo.importers.options import ImporterOptions @@ -14,6 +15,7 @@ from dojo.models import ( Engagement, Finding, + Finding_Lifecycle_Event, Test, Test_Import, ) @@ -287,6 +289,15 @@ def _process_findings_internal( # their endpoints/locations) BEFORE post_process_findings_batch # dispatches, so rules/dedup see inherited tags on .tags. apply_inherited_tags_for_findings(batch_findings) + # Provenance: one CREATED lifecycle event per new finding, bulk-written per batch + record_lifecycle_events([ + lifecycle_event( + f.id, + Finding_Lifecycle_Event.Action.CREATED, + {"test_id": self.test.id, "scan_type": self.scan_type, "kind": "import"}, + ) + for f in batch_findings + ]) batch_findings.clear() finding_ids_batch = list(batch_finding_ids) batch_finding_ids.clear() diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index 9defa8352f2..df30fd54dfd 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -13,6 +13,7 @@ find_candidates_for_deduplication_unique_id, find_candidates_for_reimport_legacy, ) +from dojo.finding.lifecycle import lifecycle_event, record_lifecycle_event, record_lifecycle_events from dojo.importers.base_importer import BaseImporter, Parser from dojo.importers.base_location_manager import LocationHandler from dojo.importers.options import ImporterOptions @@ -20,6 +21,7 @@ from dojo.models import ( Development_Environment, Finding, + Finding_Lifecycle_Event, Notes, Test, Test_Import, @@ -454,6 +456,16 @@ def _process_findings_internal( # their original creation; re-running it on no-change reimports # would be ~8 wasted queries per batch. apply_inherited_tags_for_findings(new_findings_in_batch) + # Provenance: CREATED events only for findings this reimport actually + # created — matched/unchanged findings intentionally produce no rows + record_lifecycle_events([ + lifecycle_event( + f.id, + Finding_Lifecycle_Event.Action.CREATED, + {"test_id": self.test.id, "scan_type": self.scan_type, "kind": "reimport"}, + ) + for f in new_findings_in_batch + ]) new_findings_in_batch.clear() batch_findings.clear() finding_ids_batch = list(batch_finding_ids) @@ -840,6 +852,11 @@ def process_matched_mitigated_finding( self.location_handler.record_reactivations_for_finding(existing_finding) existing_finding.notes.add(note) self.reactivated_items.append(existing_finding) + record_lifecycle_event( + existing_finding.id, + Finding_Lifecycle_Event.Action.REOPENED, + {"test_id": self.test.id, "scan_type": self.scan_type, "reason": note_entry}, + ) # The new finding is active while the existing on is mitigated. The existing finding needs to # be updated in some way # Return False here to make sure further processing happens diff --git a/dojo/jira/helper.py b/dojo/jira/helper.py index 250330618c8..7dee142e5fc 100644 --- a/dojo/jira/helper.py +++ b/dojo/jira/helper.py @@ -19,11 +19,13 @@ from dojo.celery import app from dojo.celery_dispatch import dojo_dispatch_task +from dojo.finding.lifecycle import record_lifecycle_event from dojo.forms import JIRAEngagementForm, JIRAProjectForm from dojo.models import ( Engagement, Finding, Finding_Group, + Finding_Lifecycle_Event, JIRA_Instance, JIRA_Issue, JIRA_Project, @@ -1006,6 +1008,14 @@ def failure_to_add_message(message: str, exception: Exception, _: Any) -> tuple[ j_issue.save() jira.issue(new_issue.id) logger.info("Created the following jira issue for %d:%s", obj.id, to_str_typed(obj)) + if isinstance(obj, Finding): + # Provenance: the finding's timeline shows when and as what it was ticketed + record_lifecycle_event( + obj.id, + Finding_Lifecycle_Event.Action.PUSHED_JIRA, + {"jira_key": new_issue.key}, + actor_type=Finding_Lifecycle_Event.ActorType.JIRA, + ) except Exception as e: message = f"Failed to create jira issue with the following payload: {fields} - {e}" return failure_to_add_message(message, e, obj) diff --git a/dojo/models.py b/dojo/models.py index c4d3d0aaa68..6f9662c6158 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -391,6 +391,7 @@ class Meta: BurpRawRequestResponse, # noqa: F401 -- re-export Finding, Finding_Group, # noqa: F401 -- re-export + Finding_Lifecycle_Event, # noqa: F401 -- re-export Finding_Template, Vulnerability_Id, # noqa: F401 -- re-export ) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index 4c0f161768a..b5e624c7d78 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -96,6 +96,10 @@ DD_CELERY_BROKER_PARAMS=(str, ""), DD_CELERY_BROKER_TRANSPORT_OPTIONS=(str, ""), DD_CELERY_TASK_IGNORE_RESULT=(bool, True), + # Finding lifecycle provenance events (created/closed/reopened/duplicate/pushed). + # Enabled acts as a kill switch; retention bounds table growth (purged nightly). + DD_FINDING_LIFECYCLE_EVENTS_ENABLED=(bool, True), + DD_FINDING_LIFECYCLE_EVENTS_RETENTION_DAYS=(int, 540), DD_CELERY_RESULT_BACKEND=(str, "django-db"), DD_CELERY_RESULT_EXPIRES=(int, 86400), DD_CELERY_BEAT_SCHEDULE_FILENAME=(str, root("dojo.celery.beat.db")), @@ -865,6 +869,8 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param ) CELERY_TASK_IGNORE_RESULT = env("DD_CELERY_TASK_IGNORE_RESULT") CELERY_RESULT_BACKEND = env("DD_CELERY_RESULT_BACKEND") +FINDING_LIFECYCLE_EVENTS_ENABLED = env("DD_FINDING_LIFECYCLE_EVENTS_ENABLED") +FINDING_LIFECYCLE_EVENTS_RETENTION_DAYS = env("DD_FINDING_LIFECYCLE_EVENTS_RETENTION_DAYS") CELERY_TIMEZONE = TIME_ZONE CELERY_RESULT_EXPIRES = env("DD_CELERY_RESULT_EXPIRES") CELERY_BEAT_SCHEDULE_FILENAME = env("DD_CELERY_BEAT_SCHEDULE_FILENAME") @@ -919,6 +925,13 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param "expires": int(60 * 60 * 8 * 1.2), # If a task is not executed within 9.6 hours, it should be dropped from the queue. Two more tasks should be scheduled in the meantime. }, }, + "purge-finding-lifecycle-events": { + "task": "dojo.finding.lifecycle.purge_finding_lifecycle_events", + "schedule": timedelta(hours=24), + "options": { + "expires": int(60 * 60 * 24 * 1.2), # If a task is not executed within 28.8 hours, it should be dropped from the queue. Two more tasks should be scheduled in the meantime. + }, + }, "update-findings-from-source-issues": { "task": "dojo.tools.tool_issue_updater.update_findings_from_source_issues", "schedule": timedelta(hours=3), diff --git a/unittests/test_finding_lifecycle_events.py b/unittests/test_finding_lifecycle_events.py new file mode 100644 index 00000000000..d0d241f2ba5 --- /dev/null +++ b/unittests/test_finding_lifecycle_events.py @@ -0,0 +1,205 @@ +""" +Finding lifecycle provenance events: created / closed / reopened / +marked-duplicate transitions recorded by the import pipeline and dedupe, +exposed via the finding API, bounded by the retention purge task. +""" +from datetime import timedelta + +from django.test import override_settings +from django.utils import timezone +from rest_framework.authtoken.models import Token + +from dojo.finding.lifecycle import purge_finding_lifecycle_events +from dojo.importers.default_importer import DefaultImporter +from dojo.importers.default_reimporter import DefaultReImporter +from dojo.models import ( + Development_Environment, + Engagement, + Finding, + Finding_Lifecycle_Event, + Product, + Product_Type, + System_Settings, + User, +) + +from .dojo_test_case import DojoAPITestCase, get_unit_tests_scans_path + + +class TestFindingLifecycleEvents(DojoAPITestCase): + scan_type = "Semgrep JSON Report" + + def setUp(self): + self.user, _ = User.objects.get_or_create(username="admin", defaults={"is_superuser": True, "is_staff": True}) + token, _ = Token.objects.get_or_create(user=self.user) + self.client.credentials(HTTP_AUTHORIZATION=f"Token {token.key}") + + self.environment, _ = Development_Environment.objects.get_or_create(name="Development") + product_type, _ = Product_Type.objects.get_or_create(name="lifecycle-events") + self.product, _ = Product.objects.get_or_create( + name="TestLifecycleEvents", + description="Test", + prod_type=product_type, + ) + + def _engagement(self, name): + engagement, _ = Engagement.objects.get_or_create( + name=name, + product=self.product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + return engagement + + def _import_options(self, engagement, scan_type=None): + return { + "user": self.user, + "lead": self.user, + "scan_date": None, + "environment": self.environment, + "active": True, + "verified": False, + "engagement": engagement, + "scan_type": scan_type or self.scan_type, + } + + def _events(self, finding, action=None): + qs = Finding_Lifecycle_Event.objects.filter(finding_id=finding.id).order_by("created", "id") + if action: + qs = qs.filter(action=action) + return list(qs) + + def test_full_reimport_cycle_records_created_closed_reopened(self): + engagement = self._engagement("lifecycle reimport cycle") + options = self._import_options(engagement) + + # run 1: finding at line 31 is created + with (get_unit_tests_scans_path("semgrep") / "close_old_findings_report_line31.json").open(encoding="utf-8") as scan: + importer = DefaultImporter(close_old_findings=False, **options) + test, _, len_new, _, _, _, _ = importer.process_scan(scan, force_sync=True) + self.assertEqual(1, len_new) + original = Finding.objects.filter(test=test).order_by("id").first() + + created_events = self._events(original, Finding_Lifecycle_Event.Action.CREATED) + self.assertEqual(1, len(created_events)) + self.assertEqual("import", created_events[0].detail["kind"]) + self.assertEqual(test.id, created_events[0].detail["test_id"]) + self.assertEqual(Finding_Lifecycle_Event.ActorType.IMPORT, created_events[0].actor_type) + + reimport_options = { + "test": test, + "user": self.user, + "lead": self.user, + "scan_date": None, + "minimum_severity": "Info", + "active": True, + "verified": False, + "scan_type": self.scan_type, + "close_old_findings": True, + } + + # run 2: the flaw moved to line 24 but semgrep's fingerprint is stable — + # the reimport MATCHES the existing finding. Transition-only discipline: + # an unchanged match must produce zero new lifecycle events. + with (get_unit_tests_scans_path("semgrep") / "close_old_findings_report_second_run_line24.json").open(encoding="utf-8") as scan: + reimporter = DefaultReImporter(**reimport_options) + test, _, _, len_closed, _, _, _ = reimporter.process_scan(scan, force_sync=True) + self.assertEqual(0, len_closed) + self.assertEqual(1, len(self._events(original)), "matched-unchanged reimport must not write events") + + # run 3: the tool now reports a different unique id — the old finding + # no longer matches, so it closes and a new finding is created + with (get_unit_tests_scans_path("semgrep") / "close_old_findings_report_third_run_different_unique_id.json").open(encoding="utf-8") as scan: + reimporter = DefaultReImporter(**reimport_options) + test, _, _, len_closed, _, _, _ = reimporter.process_scan(scan, force_sync=True) + self.assertEqual(1, len_closed) + + closed_events = self._events(original, Finding_Lifecycle_Event.Action.CLOSED) + self.assertEqual(1, len(closed_events)) + self.assertIn("reason", closed_events[0].detail) + self.assertEqual(test.id, closed_events[0].detail["test_id"]) + + new_finding = Finding.objects.filter(test=test).exclude(id=original.id).order_by("-id").first() + self.assertIsNotNone(new_finding) + new_created = self._events(new_finding, Finding_Lifecycle_Event.Action.CREATED) + self.assertEqual(1, len(new_created)) + self.assertEqual("reimport", new_created[0].detail["kind"]) + + # run 4: the original fingerprint reappears — the closed finding reactivates + with (get_unit_tests_scans_path("semgrep") / "close_old_findings_report_line31.json").open(encoding="utf-8") as scan: + reimporter = DefaultReImporter(**reimport_options) + reimporter.process_scan(scan, force_sync=True) + + reopened_events = self._events(original, Finding_Lifecycle_Event.Action.REOPENED) + self.assertEqual(1, len(reopened_events)) + self.assertIn("reason", reopened_events[0].detail) + + def test_dedupe_records_marked_duplicate_with_original(self): + system_settings = System_Settings.objects.get() + system_settings.enable_deduplication = True + system_settings.save() + try: + engagement = self._engagement("lifecycle dedupe") + engagement.deduplication_on_engagement = True + engagement.save() + options = self._import_options(engagement, scan_type="Acunetix Scan") + + with (get_unit_tests_scans_path("acunetix") / "one_finding.xml").open(encoding="utf-8") as scan: + importer = DefaultImporter(close_old_findings=False, **options) + _test1, _, _, _, _, _, _ = importer.process_scan(scan, force_sync=True) + with (get_unit_tests_scans_path("acunetix") / "one_finding.xml").open(encoding="utf-8") as scan: + importer = DefaultImporter(close_old_findings=False, **options) + test2, _, _, _, _, _, _ = importer.process_scan(scan, force_sync=True) + + duplicate = Finding.objects.filter(test=test2, duplicate=True).first() + self.assertIsNotNone(duplicate, "second import of the same report should produce a duplicate") + + dup_events = self._events(duplicate, Finding_Lifecycle_Event.Action.MARKED_DUPLICATE) + self.assertEqual(1, len(dup_events)) + self.assertEqual(Finding_Lifecycle_Event.ActorType.DEDUPE, dup_events[0].actor_type) + self.assertEqual(duplicate.duplicate_finding_id, dup_events[0].detail["original_id"]) + finally: + system_settings.enable_deduplication = False + system_settings.save() + + def test_lifecycle_events_api_endpoint(self): + engagement = self._engagement("lifecycle api") + options = self._import_options(engagement) + with (get_unit_tests_scans_path("semgrep") / "close_old_findings_report_line31.json").open(encoding="utf-8") as scan: + importer = DefaultImporter(close_old_findings=False, **options) + test, _, _, _, _, _, _ = importer.process_scan(scan, force_sync=True) + finding = Finding.objects.filter(test=test).first() + + response = self.client.get(f"/api/v2/findings/{finding.id}/lifecycle_events/", format="json") + self.assertEqual(200, response.status_code, response.content) + data = response.json() + self.assertTrue(data) + self.assertEqual("created", data[-1]["action"]) # newest first + self.assertEqual("import", data[-1]["detail"]["kind"]) + + def test_purge_respects_retention_window(self): + engagement = self._engagement("lifecycle purge") + options = self._import_options(engagement) + with (get_unit_tests_scans_path("semgrep") / "close_old_findings_report_line31.json").open(encoding="utf-8") as scan: + importer = DefaultImporter(close_old_findings=False, **options) + importer.process_scan(scan, force_sync=True) + + total_before = Finding_Lifecycle_Event.objects.count() + self.assertGreater(total_before, 0) + + # nothing is old enough: purge deletes nothing + self.assertEqual(0, purge_finding_lifecycle_events()) + + # backdate everything past the retention window: purge removes it all + Finding_Lifecycle_Event.objects.update(created=timezone.now() - timedelta(days=9999)) + self.assertEqual(total_before, purge_finding_lifecycle_events()) + self.assertEqual(0, Finding_Lifecycle_Event.objects.count()) + + @override_settings(FINDING_LIFECYCLE_EVENTS_ENABLED=False) + def test_kill_switch_disables_event_writes(self): + engagement = self._engagement("lifecycle disabled") + options = self._import_options(engagement) + with (get_unit_tests_scans_path("semgrep") / "close_old_findings_report_line31.json").open(encoding="utf-8") as scan: + importer = DefaultImporter(close_old_findings=False, **options) + importer.process_scan(scan, force_sync=True) + self.assertEqual(0, Finding_Lifecycle_Event.objects.count()) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 47d30a99824..48278c0a0c8 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -343,13 +343,13 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=157, + expected_num_queries1=158, expected_num_async_tasks1=2, - expected_num_queries2=122, + expected_num_queries2=123, expected_num_async_tasks2=1, expected_num_queries3=29, expected_num_async_tasks3=1, - expected_num_queries4=100, + expected_num_queries4=106, expected_num_async_tasks4=0, ) @@ -367,13 +367,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=173, + expected_num_queries1=174, expected_num_async_tasks1=2, - expected_num_queries2=130, + expected_num_queries2=131, expected_num_async_tasks2=1, expected_num_queries3=37, expected_num_async_tasks3=1, - expected_num_queries4=100, + expected_num_queries4=106, expected_num_async_tasks4=0, ) @@ -392,13 +392,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=183, + expected_num_queries1=184, expected_num_async_tasks1=4, - expected_num_queries2=140, + expected_num_queries2=141, expected_num_async_tasks2=3, expected_num_queries3=44, expected_num_async_tasks3=3, - expected_num_queries4=109, + expected_num_queries4=115, expected_num_async_tasks4=2, ) @@ -524,9 +524,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=93, + expected_num_queries1=94, expected_num_async_tasks1=2, - expected_num_queries2=73, + expected_num_queries2=74, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -545,9 +545,9 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=109, + expected_num_queries1=110, expected_num_async_tasks1=2, - expected_num_queries2=90, + expected_num_queries2=97, expected_num_async_tasks2=2, ) @@ -633,13 +633,13 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=164, + expected_num_queries1=165, expected_num_async_tasks1=2, - expected_num_queries2=131, + expected_num_queries2=132, expected_num_async_tasks2=1, expected_num_queries3=37, expected_num_async_tasks3=1, - expected_num_queries4=101, + expected_num_queries4=107, expected_num_async_tasks4=0, ) @@ -657,13 +657,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=182, + expected_num_queries1=183, expected_num_async_tasks1=2, - expected_num_queries2=141, + expected_num_queries2=142, expected_num_async_tasks2=1, expected_num_queries3=47, expected_num_async_tasks3=1, - expected_num_queries4=101, + expected_num_queries4=107, expected_num_async_tasks4=0, ) @@ -682,13 +682,13 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=195, + expected_num_queries1=196, expected_num_async_tasks1=4, - expected_num_queries2=154, + expected_num_queries2=155, expected_num_async_tasks2=3, expected_num_queries3=54, expected_num_async_tasks3=3, - expected_num_queries4=113, + expected_num_queries4=119, expected_num_async_tasks4=2, ) @@ -789,9 +789,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=100, + expected_num_queries1=101, expected_num_async_tasks1=2, - expected_num_queries2=76, + expected_num_queries2=77, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -809,8 +809,8 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=118, + expected_num_queries1=119, expected_num_async_tasks1=2, - expected_num_queries2=201, + expected_num_queries2=208, expected_num_async_tasks2=2, ) diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 698b1126a85..cfeac004753 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -590,9 +590,9 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # the async watson indexer, executed inline under CELERY_TASK_ALWAYS_EAGER); # +5 reimport (no-change + with-new) queries from removal of # WATSON_ASYNC_INDEX_UPDATE_THRESHOLD making async dispatch unconditional. - EXPECTED_ZAP_IMPORT_V2 = 287 - EXPECTED_ZAP_IMPORT_V3 = 311 + EXPECTED_ZAP_IMPORT_V2 = 288 + EXPECTED_ZAP_IMPORT_V3 = 312 EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 74 EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 86 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 148 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 177 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 149 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 178