­­­­­­­­­­­­­­­­­­
<!DOCTYPE html>
<html>
"""
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.


This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
See the GNU General Public License for more details.


You should have received a copy of the GNU General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.

Copyright © 2019 Cloud Linux Software Inc.

This software is also available under ImunifyAV commercial license,
see <https://www.imunify360.com/legal/eula>
"""
import asyncio
import os
import time
from datetime import datetime, timedelta
from logging import getLogger

from defence360agent.api import inactivity
from defence360agent.contracts.messages import MessageType
from defence360agent.contracts.plugins import Scope
from defence360agent.internals.global_scope import g
from defence360agent.mr_proper import BaseCleaner
from defence360agent.subsys import persistent_state
from defence360agent.utils import is_cluster, nice_iterator, split_for_chunk
from imav.malwarelib.config import (
    MalwareHitStatus,
    MalwareScanResourceType,
    MalwareScanType,
)
from imav.malwarelib.model import MalwareHistory, MalwareHit, MalwareScan
from imav.malwarelib.scan.utils import safe_realpath
from imav.malwarelib.subsys.malware import MalwareAction
from imav.subsys import realtime_av

logger = getLogger(__name__)


class OutdatedScansCleaner(BaseCleaner):
    PERIOD = timedelta(days=1).total_seconds()
    CLEANUP_LIMIT_DELTA = timedelta(days=30)
    LOCK_FILE = persistent_state.register_lock_file(
        "malware-scans-cleaner", Scope.AV_IM360
    )

    @classmethod
    async def cleanup(cls) -> None:
        cleanup_time_limit = int(
            (datetime.now() - cls.CLEANUP_LIMIT_DELTA).timestamp()
        )
        deleted = (
            MalwareScan.delete()
            .where(MalwareScan.started < cleanup_time_limit)
            .execute()
        )
        logger.info("Cleaned %s outdated scans", deleted)


class OutdatedHistoryCleaner(BaseCleaner):
    PERIOD = timedelta(days=1).total_seconds()
    CLEANUP_LIMIT_DELTA = timedelta(days=30)
    LOCK_FILE = persistent_state.register_lock_file(
        "malware-history-cleaner", Scope.AV_IM360
    )

    @classmethod
    async def cleanup(cls) -> None:
        keep_time_threshold = int(
            (datetime.now() - cls.CLEANUP_LIMIT_DELTA).timestamp()
        )
        deleted = (
            MalwareHistory.delete()
            .where(MalwareHistory.ctime < keep_time_threshold)
            .execute()
        )
        logger.info("Cleaned %s outdated malware history", deleted)


class OutdatedHitsCleaner(BaseCleaner):
    PERIOD = int(
        os.getenv(
            "IMUNIFY360_OUTDATED_HITS_CHECK_INTERVAL",
            timedelta(days=1).total_seconds(),
        )
    )
    FILE_NAME = os.getenv(
        "MALWARE_HITS_CLEANER_LOCK_FILE", "malware-hits-cleaner"
    )
    LOCK_FILE = persistent_state.register_lock_file(FILE_NAME, Scope.AV_IM360)
    REALTIME_SCAN_THRESHOLD = timedelta(minutes=10).total_seconds()
    CHUNK_SIZE = 1000

    @classmethod
    async def _cleanup(cls) -> None:
        """Rescan irrelevant malware hits"""
        to_rescan = []
        not_exist_hits = []
        # re-check only *malicious* hits
        hits = (
            MalwareHit.select()
            .where(
                MalwareHit.status == MalwareHitStatus.FOUND,
                MalwareHit.resource_type == MalwareScanResourceType.FILE.value,
                MalwareHit.malicious == True,  # noqa: E712
            )
            .order_by(MalwareHit.timestamp.asc())
        )
        async for hit in nice_iterator(hits, chunk_size=cls.CHUNK_SIZE):
            # orig_file_path is a TenantPath: standalone stats the local file;
            # in k8s it routes through the syncer so we read the tenant pod's
            # filesystem, not the agent container's (where the file is absent
            # and every hit would otherwise look deleted).
            tenant_path = hit.orig_file_path
            try:
                st = await tenant_path.stat_async()
            except FileNotFoundError:
                # Definitively gone (local stat, or the syncer reported
                # missing) — safe to drop the stale hit.
                not_exist_hits.append(hit.id)
                continue
            except (OSError, ValueError) as exc:
                # OSError: transient (gateway down, permission). ValueError: a
                # tenantless path we cannot resolve in cluster mode. We could
                # NOT verify the file, so keep the hit rather than delete a
                # finding we have not confirmed is gone.
                logger.warning(
                    "Can't check %r due to %s; keeping hit", tenant_path, exc
                )
                continue
            # ctime locally; the k8s syncer exposes only mtime, an equivalent
            # "changed since the scan" signal.
            changed_at = st.st_mtime if tenant_path.is_remote else st.st_ctime
            if hit.timestamp < changed_at:
                # rescan the modified files after scanning,
                # they may not be infected anymore
                realtime_threshold = (
                    time.time() - cls.REALTIME_SCAN_THRESHOLD
                )  # don't scan file twice
                if (
                    not realtime_av.should_be_running()
                    or changed_at < realtime_threshold
                ):
                    to_rescan.append(hit.orig_file)
        if to_rescan:
            for files in split_for_chunk(to_rescan, chunk_size=cls.CHUNK_SIZE):
                logger.info("Rescan %s outdated malware files", len(files))
                await g.sink.process_message(
                    MessageType.MalwareRescanFiles(
                        files=files, type=MalwareScanType.RESCAN_OUTDATED
                    )
                )
        # delete db entries for non-existent files
        for hits_to_delete in split_for_chunk(
            not_exist_hits, chunk_size=cls.CHUNK_SIZE
        ):
            hits = list(
                MalwareHit.select()
                .where(MalwareHit.id.in_(hits_to_delete))
                .where(MalwareHit.status == MalwareHitStatus.FOUND)
                .where(MalwareHit.malicious == True)  # noqa: E712
            )
            deleted = (
                MalwareHit.delete().where(MalwareHit.id.in_(hits)).execute()
            )
            await MalwareAction.not_exist(
                hits, cause=MalwareScanType.RESCAN_OUTDATED
            )
            logger.info("Deleted %s not exist malware hits", deleted)
            # don't block the whole loop for too long,
            # return control to the loop every iteration
            await asyncio.sleep(0)

    @classmethod
    async def cleanup(cls):
        with inactivity.track.task("malware hits relevance check"):
            await cls._cleanup()


class OutdatedDbHitsCleaner(BaseCleaner):
    PERIOD = timedelta(days=1).total_seconds()
    LOCK_FILE = persistent_state.register_lock_file(
        "malware-db-hits-cleaner", Scope.AV_IM360
    )
    CHUNK_SIZE = 1000

    @staticmethod
    def _app_path_is_gone(path: str) -> bool:
        # gone only if both symlink forms are missing; other errors != gone
        real = safe_realpath(path)
        if real is None:
            return False
        forms = (path, real)
        for form in forms:
            try:
                os.stat(form)
                return False
            except FileNotFoundError:
                continue
            except (OSError, ValueError):
                return False
        return True

    @classmethod
    async def _cleanup(cls) -> None:
        if is_cluster():
            # tenant-prefixed paths are invisible here; stat would retire all
            return
        hits = MalwareHit.select().where(
            MalwareHit.status == MalwareHitStatus.FOUND,
            MalwareHit.resource_type == MalwareScanResourceType.DB.value,
        )
        gone = []
        async for hit in nice_iterator(hits, chunk_size=cls.CHUNK_SIZE):
            if cls._app_path_is_gone(hit.orig_file):
                gone.append(hit.id)
        deleted = 0
        for chunk in split_for_chunk(gone, chunk_size=cls.CHUNK_SIZE):
            # a collected hit may leave FOUND before the delete runs
            chunk_hits = list(
                MalwareHit.select().where(
                    MalwareHit.id.in_(chunk),
                    MalwareHit.status == MalwareHitStatus.FOUND,
                    MalwareHit.resource_type
                    == MalwareScanResourceType.DB.value,
                )
            )
            deleted += (
                MalwareHit.delete()
                .where(MalwareHit.id.in_(chunk_hits))
                .execute()
            )
            await MalwareAction.not_exist(
                chunk_hits, cause=MalwareScanType.RESCAN_OUTDATED
            )
            await asyncio.sleep(0)
        if deleted:
            logger.info("Deleted %s db hits with missing app paths", deleted)

    @classmethod
    async def cleanup(cls):
        with inactivity.track.task("db malware hits relevance check"):
            await cls._cleanup()