import html
import io
from typing import Optional, Set, Dict, List, Tuple
from dataclasses import dataclass, field

from tqdm import tqdm
from dataclasses_json import dataclass_json
import json
import os
import pathlib
import shlex
from loguru import logger
from enum import Enum
import sys

from flask import Blueprint, render_template, request, redirect, url_for, current_app, send_file
from werkzeug.utils import secure_filename

import config
from helpers.disk_file_versioning import PROFILES_FILEPATH
from helpers.object_size import get_size
from helpers.flask_response import basic_msg
from helpers.time_helper import current_timestamp
from helpers.md_and_html_convertor import MdAndHTMLConvertor
import helpers.table_of_services
from helpers.file_utils import relative_path, extract_fid_from_filename, json_safe_load, generate_random_folder_name, \
    persistent_audit_scopes_file_path, folder_in_upload_path
from template_manager import PwndocTemplateManager
from helpers.flask_multifile_upload import save_multiple_uploaded_files
from scan2report import scan2report
from api_templates import download_templates_from_pwndoc_to_scan2report
from pwndoc_api import get_findings_from_audit, upsert_raw_finding, get_audits, does_audit_exist, get_audit_language
from template_scan2report import TemplateScan2Report, str_bool_to_bool
from template_pwndoc import get_fid_from_raw_finding
from template_converters import Scan2ReportConvert
import template_converters
from helpers.custom_logging import FlashLog  # Beware that using FlashLog might not show, if used inside background worker.
from helpers.rq_helper import REDIS_QUEUE, rq_job_finalized, rq_job_failed
import redis  # dependency via rq


INFO_FILE = "_pwndoc_importer_processing.json"


class OutputFormat(Enum):
    PWNDOC = "pwndoc"
    JSON = "json"
    CSV = "csv"
    DOCX = "docx"


class ProcessingSteps(Enum):
    UPLOAD_VULN_SCANS = "Upload scan results from browser to server"
    PREPARATIONS_FOR_SCAN2REPORT = "Download templates from PwnDoc"
    SCAN2REPORT_FIRST_PASS = "Process scan findings using Scan2Report"
    UPLOAD_OF_NEW_TEMPLATES = "Upload newly found templates to PwnDoc"
    SCAN2REPORT_SECOND_PASS = "Rerun Scan2Report with new templates (if any)"
    PWNDOC_FINDINGS_UPLOAD = "Upload findings to PwnDoc audit"


class ProcessingStatus(Enum):
    WAITING = '⏳',
    RUNNING = '🚀',
    DONE = '✔',
    FAILED = '❌',
    SKIPPED = '⏩'


FINISHED_STATES = {ProcessingStatus.DONE, ProcessingStatus.FAILED, ProcessingStatus.SKIPPED}


SEVERITIES = {  # todo: synchronize this with scan2report
    '0': 'Informative',
    '1': 'Low',
    '2': 'Medium',
    '3': 'High',
    '4': 'Critical'
}


# TODO: There is race condition. When two people are adding findings - the scan2report template synchronization might break the other scan2report run.

bp = Blueprint('api_process_findings', __name__, template_folder='templates')


def get_audits_with_nice_names(include_file_formats: bool = True) -> Dict[str, str]:
    audits_dict = {}
    for audit_id, audit_name in get_audits():
        nice_name = f'{audit_name}  (id: {audit_id})'
        audits_dict[nice_name] = audit_id

    if include_file_formats:
        audits_dict.update({
            "Export to JSON": OutputFormat.JSON.value,
            "Export to CSV": OutputFormat.CSV.value,
            "Export to DOCX": OutputFormat.DOCX.value,
        })

    return audits_dict


def _list_profiles() -> List[str]:
    with open(PROFILES_FILEPATH, encoding="utf8") as f:
        data: dict = json.load(f)
    return list(data.keys())


@bp.route("/upload_scanner_result/<audit_id>", methods=["GET"])
@bp.route("/upload_scanner_result/", methods=["GET"])
@bp.route("/upload_scanner_result", methods=["GET"])
def upload_scanner_result_get(audit_id: str = ""):  # todo: maybe add additional arguments for setting persistence across error
    audits_dict = get_audits_with_nice_names()
    if len(audit_id) and audit_id not in list(audits_dict.values()):
        FlashLog.warning("Audit ID from URL doesn't exist.")
        audit_id = OutputFormat.DOCX.value

    languages = list(filter(lambda x: x != "og", config.LOCALES))
    profiles = _list_profiles()

    return render_template('findings_import.html', audit_dict=audits_dict, audit_id=audit_id, lang_list=languages, severities=SEVERITIES, profiles=profiles, allowed_scanner_extensions=config.ALLOWED_SCANNER_EXTENSIONS)


def _go_to_start(audit_id: str = ""):
    return redirect(url_for('api_process_findings.upload_scanner_result_get', audit_id=audit_id))


def _go_to_meta(folder_name: str):
    return redirect(url_for("api_process_findings.show_meta_output", folder_name=folder_name))


def _flash_msg_and_go_to_start(msg: str, level: str = "warning", audit_id: str = ""):
    FlashLog.log_and_flash_with_level(msg, level, depth=2)
    # TODO: ps.user_msgs
    return _go_to_start(audit_id)


@dataclass_json
@dataclass
class ProcessingSettings:
    audit_id: str = OutputFormat.DOCX.value
    upload_to_pwndoc: bool = False
    scan2report_args: str = ''
    __folder_name: str = field(default_factory=lambda: generate_random_folder_name("findings_import"))
    stored_files: Set[str] = field(default_factory=set)
    locale: str = 'cs'

    steps_progress: Dict[str, Tuple[str, int, int]] = field(default_factory=lambda:
        {e.name: (ProcessingStatus.WAITING.name, 0, 0) for e in ProcessingSteps}
    )

    user_msgs: List[Tuple[str, str]] = field(default_factory=list)  # level (info, warning, error), msg
    finished: bool = False

    missing_templates: List[str] = field(default_factory=list)  # todo: missing templates might not include grouped findings which don't have their own fid in the resulting findings.json
    output_unspecified: dict = field(default_factory=dict)

    def add_msg(self, level: str, msg: str):
        """
           :param str level: info, warning, error
        """
        if level not in ['info', 'warning', 'error']:
            logger.warning(f'Adding ProcessingSettings with incorrect level: {level}')
        self.user_msgs.append((level, msg))
        self.save()

    @staticmethod
    def __get_folder_path(folder_name: str) -> str:
        assert folder_name == secure_filename(folder_name)
        return folder_in_upload_path(folder_name)

    def get_folder_path(self) -> str:
        return self.__get_folder_path(self.__folder_name)

    def get_folder_name(self) -> str:
        return self.__folder_name

    @classmethod
    def __get_info_file_path(cls, folder_name: str) -> str:
        return os.path.join(cls.__get_folder_path(folder_name), INFO_FILE)

    @classmethod
    def load_from_folder_name(cls, folder_name: str) -> Optional['ProcessingSettings']:
        filepath = cls.__get_info_file_path(folder_name)
        if not pathlib.Path(filepath).is_file():
            return None

        # todo: here can occur a race condition - concurrent write and read resulting in invalid read of JSON
        with open(filepath, encoding="utf-8") as f:
            file_data = json.load(f)
            return ProcessingSettings.from_dict(file_data)

    def save(self):
        pathlib.Path(self.get_folder_path()).mkdir(exist_ok=True)
        with open(self.__get_info_file_path(self.__folder_name), "w", encoding="utf-8") as f:
            json.dump(self.to_dict(), f, indent=4)

    def cleanup_uploaded_files(self):
        files = [os.path.join(self.get_folder_path(), f) for f in self.stored_files]

        for file in files:
            if pathlib.Path(file).is_file():
                os.remove(file)

    def get_result_path(self) -> str:
        if self.audit_id == OutputFormat.DOCX.value:
            return self._get_result_docx_path()
        if self.audit_id == OutputFormat.CSV.value:
            return self._get_result_csv_path()
        return self._get_result_json_path()

    def _get_result_docx_path(self):
        return os.path.join(self.get_folder_path(), 'scan2report_output.docx')

    def _get_result_json_path(self):
        return os.path.join(self.get_folder_path(), 'scan2report_output.json')

    def _get_result_csv_path(self):
        return os.path.join(self.get_folder_path(), 'scan2report_output.csv')

    def _meta_file_path(self) -> str:
        return f"{self.get_result_path()}.meta.json"

    def load_meta_file(self) -> dict:
        meta_data = json_safe_load(self._meta_file_path())
        return meta_data if meta_data else {}

    def set_step_status(self, step: ProcessingSteps, new_status: ProcessingStatus):
        # todo: make sure all calls to this function happen even when exceptions occur
        step_name = step.name
        old_status_name, start_timestamp, end_timestamp = self.steps_progress[step_name]

        if old_status_name == ProcessingStatus.WAITING.name:
            start_timestamp = current_timestamp()

        if new_status in FINISHED_STATES:
            end_timestamp = current_timestamp()
            if start_timestamp == 0:
                start_timestamp = end_timestamp

        if new_status == ProcessingStatus.FAILED:
            self.finished = True

        self.steps_progress[step_name] = (new_status.name, start_timestamp, end_timestamp)

        self.fail_remaining_steps_if_finished()  # beware this is recursive - its implementation is calling this function to set other steps to FAILED
        self.save()

    def get_statuses_for_display(self) -> List[Tuple[str, str, str]]:
        """Step display text | Step status text | Step duration in minutes and seconds"""

        answer: List[Tuple[str, str, str]] = []

        for step_enum in ProcessingSteps:
            step_info = self.steps_progress.get(step_enum.name, None)
            if step_info is None:
                continue
            old_status_name, start_timestamp, end_timestamp = step_info

            if end_timestamp >= start_timestamp:
                duration = end_timestamp - start_timestamp
            elif start_timestamp != 0:
                duration = current_timestamp() - start_timestamp
            else:
                duration = 0

            step_text = step_enum.value

            step_status = ProcessingStatus[step_info[0]]
            step_status_text = step_status.value[0] if isinstance(step_status.value, tuple) else step_status.value  # todo: inspect why python sees some emojis as tuples

            step_duration_text = ""
            if step_status == ProcessingStatus.RUNNING or step_status in FINISHED_STATES:
                step_duration_text = f"{str(duration//60).zfill(2)}:{str(duration%60).zfill(2)}"

            answer.append((step_text, step_status_text, step_duration_text))

        return answer

    def rq_job_name(self):
        return self.get_folder_name()

    def rq_is_still_in_enqueued(self) -> bool:
        return rq_job_finalized(self.rq_job_name()) is False

    def rq_job_failed(self) -> bool:
        return rq_job_failed(self.rq_job_name())

    def fail_remaining_steps_if_finished(self):
        if not self.finished:
            return

        for step_name, step_value in self.steps_progress.items():
            if step_value[0] not in [x.name for x in FINISHED_STATES]:
                self.set_step_status(ProcessingSteps[step_name], ProcessingStatus.FAILED)


@bp.route("/upload_scanner_result", methods=["POST"])
def upload_scanner_result_post():
    audit_id = request.form.get("audit")
    locale = request.form.get("locale")
    profile = request.form.get("profile")
    min_severity = request.form.get("min-severity")

    def _flash_error_msg(msg: str, level: str = "warning"):
        return _flash_msg_and_go_to_start(msg, level, audit_id)  # note: audit_id is taken from outer scope

    if len(audit_id) and audit_id not in list(get_audits_with_nice_names().values()):
        return _flash_error_msg("This audit ID doesn't exist.")

    if locale not in config.LOCALES:
        return _flash_error_msg("This locale does not exist.")

    if profile not in _list_profiles():
        return _flash_error_msg("This profile does not exist.")

    if min_severity not in SEVERITIES.keys():
        return _flash_error_msg("This severity doesn't exist")

    scan2report_args = request.form.get("scan2report_args")
    scan2report_args += f' --min-severity {min_severity} --profile {profile}'  # todo: make sure that I'm not overwriting explicitly set parameters

    upload_to_pwndoc = does_audit_exist(audit_id)

    ps = ProcessingSettings(
        audit_id=audit_id,
        scan2report_args=scan2report_args,
        locale=locale,
        upload_to_pwndoc=upload_to_pwndoc,
    )
    ps.set_step_status(ProcessingSteps.UPLOAD_VULN_SCANS, ProcessingStatus.RUNNING)
    if not upload_to_pwndoc:
        ps.set_step_status(ProcessingSteps.PWNDOC_FINDINGS_UPLOAD, ProcessingStatus.SKIPPED)

    error_msg, saved_filenames = save_multiple_uploaded_files(output_folder=ps.get_folder_path(), reserved_filenames=set(INFO_FILE), allowed_extensions=config.ALLOWED_SCANNER_EXTENSIONS)
    ps.stored_files = ps.stored_files.union(saved_filenames)
    ps.save()

    if len(error_msg) > 0:
        ps.set_step_status(ProcessingSteps.UPLOAD_VULN_SCANS, ProcessingStatus.FAILED)
        ps.cleanup_uploaded_files()
        return _flash_msg_and_go_to_start(error_msg, "error", audit_id)

    ps.set_step_status(ProcessingSteps.UPLOAD_VULN_SCANS, ProcessingStatus.DONE)

    if config.RQ_DISABLED:
        full_processing_native(ps.get_folder_name())
    else:
        try:
            REDIS_QUEUE.enqueue(
                full_processing_background_wrapper, args=(ps.get_folder_name(),),
                job_timeout=config.RQ_JOB_TIMEOUT, ttl='1d', result_ttl=7*24*60*60, job_id=ps.rq_job_name()
            )
        except redis.exceptions.ConnectionError as e:
            FlashLog.log_and_flash_with_level(f"Failed to connect to Redis - is the container running? ({e})", "error")
    return _go_to_meta(ps.get_folder_name())


def _upload_missing_templates(ps: ProcessingSettings):
    PwndocTemplateManager.update_template_db_from_pwndoc()  # Update DB so that we can do automatic template merging.

    for single_missing_string in tqdm(ps.missing_templates, desc="Uploading missing templates"):
        filename, severity, title = single_missing_string.strip().split("\t")
        filename = secure_filename(filename)
        fid = extract_fid_from_filename(filename)
        path = os.path.join(TemplateScan2Report.native_folder_name_for_locale(ps.locale, original=True), filename)
        with open(path, encoding="utf8") as f:
            s2r_template_dict = json.load(f)

        og_template: TemplateScan2Report = TemplateScan2Report.parse_dict_custom(fid=fid, locale="og", template_dict=s2r_template_dict, original=True)
        if og_template is None:
            logger.info(f"Empty og template for fid {fid}?")
            continue
        og_template_tns = Scan2ReportConvert.convert_single_finding_to_tns_pwndoc(og_template, "og")

        # PoC and Hosts are automatically removed from template before it's uploaded

        try:
            PwndocTemplateManager.add_single_locale_template(og_template_tns)
        except AssertionError as e:
            logger.error(f'Upload of template failed.', stack_info=True, exc_info=True)
            ps.add_msg('error', f"Upload of template with FID {fid} failed. This shouldn't happen, but to not totally block the import it was skipped.")

    TemplateScan2Report.delete_template_files(["og"])
    PwndocTemplateManager.update_template_db_from_pwndoc()  # Update DB to have a current state.


def get_pwndoc_audit_locale(ps: ProcessingSettings) -> str:
    if not ps.upload_to_pwndoc:
        return ps.locale
    return get_audit_language(ps.audit_id)


def process_scanner_result_using_scan2report(folder_name: str) -> bool:
    ps: Optional[ProcessingSettings] = ProcessingSettings.load_from_folder_name(folder_name)
    if ps is None:
        logger.warning(f"Scan folder doesn't exist: {folder_name}")
        return False

    ps.set_step_status(ProcessingSteps.PREPARATIONS_FOR_SCAN2REPORT, ProcessingStatus.RUNNING)

    download_templates_from_pwndoc_to_scan2report(
        "scan2report_native" if ps.audit_id == OutputFormat.DOCX.value else "scan2report_with_pwndoc_formatting",
        locale=ps.locale
    )

    files = [os.path.join(ps.get_folder_path(), f) for f in ps.stored_files]
    args = shlex.split(ps.scan2report_args)

    if "--output" in args:
        ps.add_msg('error', "Don't specify --output as a parameter, use selector of audit ID instead.")
        return False

    if "-l" in args or "--lang" in args:
        ps.add_msg('error', "Don't specify --lang as a parameter, use selector instead.")
        return False

    s2r_config = scan2report.Scan2ReportConfig()
    s2r_config.rootDir = config.SCAN2REPORT_ALTERNATIVE_FOLDER
    s2r_config.lang = ps.locale
    s2r_config.outfile = ps.get_result_path()

    pwndoc_audit_locale = get_pwndoc_audit_locale(ps)
    if ps.locale != pwndoc_audit_locale:
        ps.add_msg('warning', f"Warning - the locale of the import ({ps.locale}) doesn't match the audits locale ({pwndoc_audit_locale}).")

    combined_args = args + files
    logger.debug(f"Running scan2report with: {combined_args} and passing additional options as part of config struct")

    # todo: the http connection might time out - do as async?
    count_of_known_templates_before = len(TemplateScan2Report.list_files_in_locale_folder(ps.locale, original=False))
    ps.set_step_status(ProcessingSteps.PREPARATIONS_FOR_SCAN2REPORT, ProcessingStatus.DONE)
    ps.set_step_status(ProcessingSteps.SCAN2REPORT_FIRST_PASS, ProcessingStatus.RUNNING)

    scan2report.main(combined_args, s2r_c=s2r_config)  # First run to generates OG templates

    # Metadata from the first load - extract the missing templates.
    meta_data = ps.load_meta_file()
    ps.missing_templates = meta_data["missing"]  # _upload_missing_templates
    ps.output_unspecified["missing_n_templates"] = len(ps.missing_templates)
    ps.save()

    ps.set_step_status(ProcessingSteps.SCAN2REPORT_FIRST_PASS, ProcessingStatus.DONE)
    ps.set_step_status(ProcessingSteps.UPLOAD_OF_NEW_TEMPLATES, ProcessingStatus.RUNNING)

    _upload_missing_templates(ps)
    download_templates_from_pwndoc_to_scan2report(
        output_format="scan2report_native" if ps.audit_id == OutputFormat.DOCX.value else "scan2report_with_pwndoc_formatting",
        locale='og',
        force_locale_folder=ps.locale,
        prefix_title_with_locale=True
    )
    ps.set_step_status(ProcessingSteps.UPLOAD_OF_NEW_TEMPLATES, ProcessingStatus.DONE)

    count_of_known_templates_after = len(TemplateScan2Report.list_files_in_locale_folder(ps.locale, original=False))
    if count_of_known_templates_before != count_of_known_templates_after:
        ps.set_step_status(ProcessingSteps.SCAN2REPORT_SECOND_PASS, ProcessingStatus.RUNNING)
        scan2report.main(combined_args, s2r_c=s2r_config)  # Second run to regenerate the findings if during the first run we found OG templates
        ps.set_step_status(ProcessingSteps.SCAN2REPORT_SECOND_PASS, ProcessingStatus.DONE)
    else:
        ps.set_step_status(ProcessingSteps.SCAN2REPORT_SECOND_PASS, ProcessingStatus.SKIPPED)

    ps.cleanup_uploaded_files()
    return True


def combine_with_existing_finding(ps: ProcessingSettings, new_finding: dict, existing_raw_findings: List[dict]) -> Tuple[dict, bool]:
    fid = get_fid_from_raw_finding(new_finding)

    # Todo: should the conversion be here? Remove it?
    for attr_name in template_converters.HTML_ATTRIBUTES:
        new_finding[attr_name] = MdAndHTMLConvertor.html_to_pwndoc_html(new_finding.get(attr_name, ""))

    existing_raw_finding_with_dict = list(filter(lambda x: get_fid_from_raw_finding(x) == fid, existing_raw_findings))
    if len(existing_raw_finding_with_dict) == 0:
        return new_finding, True

    # From this point on we're aiming to modify the old_finding, NOT the new_finding.

    if len(existing_raw_finding_with_dict) >= 2:
        ps.add_msg('warning', f"There are already multiple findings with fid {fid} in a single audit. That is not supported.")

    old_finding = existing_raw_finding_with_dict[0]

    def calculate_new_scope(new_finding: dict, old_finding: dict) -> Tuple[str, bool]:
        original_scope = old_finding.get("scope", "").split("<br>")
        new_scope_partial = new_finding.get("scope", "").split("<br>")

        merged_scope = original_scope + new_scope_partial
        merged_scope_deduplicated = list(dict.fromkeys(merged_scope))

        new_scope_str = "<br>".join(merged_scope_deduplicated)
        changed = len(original_scope) != len(merged_scope_deduplicated)
        return new_scope_str, changed

    new_scope, scope_changed = calculate_new_scope(new_finding, old_finding)
    if not scope_changed:
        return old_finding, False

    # todo: should scope be sorted?
    old_finding["scope"] = new_scope
    old_finding["poc"] = old_finding.get("poc", "") + "<br><br>=====<br><br>" + new_finding.get("poc", "")
    return old_finding, True


def _extend_audit_services(audit_id: str, new_services: dict) -> helpers.table_of_services.Scope:
    answer = helpers.table_of_services.get_tns_table_of_services(audit_id)

    persistent_scopes_file_path = persistent_audit_scopes_file_path(audit_id)
    local_services_copy = json_safe_load(persistent_scopes_file_path)
    if local_services_copy is not None:
        for host in local_services_copy.get('scope', {}).get('hosts', []):
            for service in host.get('services', []):
                answer.add_entry(host['ip'], service['port'], service['protocol'], service['name'])

    for host_ip, port_list in new_services.items():
        for port_number, l3_protocol_list in port_list.items():
            for l3_protocol, app_protocol in l3_protocol_list.items():
                answer.add_entry(host_ip, port_number, l3_protocol, app_protocol)
    return answer


@bp.route("/force_upload_services/<audit_id>", methods=["GET"])
def force_upload_services(audit_id: str):
    services_as_pwndoc_scope = _extend_audit_services(audit_id, {})
    helpers.table_of_services.set_tns_table_of_services(audit_id, services_as_pwndoc_scope)
    return basic_msg(f"OK; {len(services_as_pwndoc_scope.hosts)} hosts are in the table of services")


def upload_scan_findings_to_pwndoc(folder_name: str) -> bool:
    ps: Optional[ProcessingSettings] = ProcessingSettings.load_from_folder_name(folder_name)
    if ps is None:
        logger.warning(f"Scan folder doesn't exist: {folder_name}")
        return False

    ps.set_step_status(ProcessingSteps.PWNDOC_FINDINGS_UPLOAD, ProcessingStatus.RUNNING)

    path = ps.get_result_path()
    with open(path, encoding="utf8") as f:
        file_data = json.load(f)

    findings: List[TemplateScan2Report] = []
    locale = ps.locale

    for single_finding in file_data:
        finding_parsed = TemplateScan2Report.parse_dict_custom(single_finding["fid"], locale, single_finding)
        if finding_parsed is None:
            logger.warning(f"Empty finding in audit report?")
        else:
            findings.append(finding_parsed)

    existing_raw_findings = get_findings_from_audit(ps.audit_id)
    new_or_changed_count = 0

    findings.sort(key=lambda x: -1 if get_size(x) < 10**6 else get_size(x))  # place big findings (over 1MB) at the end, to ensure one big findings doesnt block them all unnecessarily; sort has a stable algorithm

    not_changed_findings = set()
    for single_finding in tqdm(findings, desc="Converting findings to PwnDoc format and upserting"):
        try:
            new_or_changed = upload_single_finding_to_pwndoc(single_finding, locale, ps, existing_raw_findings)
            if new_or_changed:
                new_or_changed_count += 1
            else:
                not_changed_findings.add(single_finding.fid)
        except Exception as e:
            logger.exception("Exception on PwnDoc processing and upload")
            ps.add_msg('error', f'Failed to process and upload finding {single_finding.fid}.')

    upload_open_ports_to_pwndoc(ps)

    if len(not_changed_findings) > 0:
        ps.add_msg('info', f"The following findings were not updated, because their scope did not expand: {not_changed_findings}")
        # For those where scope is expanded, only PoC is modified.

    ps.set_step_status(ProcessingSteps.PWNDOC_FINDINGS_UPLOAD, ProcessingStatus.DONE)
    ps.add_msg('info', f'OK, added or updated {new_or_changed_count} findings - you can now find them in PwnDoc.')
    return True


def upload_single_finding_to_pwndoc(single_finding: TemplateScan2Report, locale: str, ps: ProcessingSettings, existing_raw_findings) -> bool:
    single_finding_pwndoc = Scan2ReportConvert.convert_single_finding_to_tns_pwndoc(single_finding, locale)
    single_finding_pwndoc.to_pwndoc_html()
    single_finding_pwndoc.set_custom_field("original_description_from_scan", single_finding_pwndoc.description)
    single_finding_pwndoc_dict = single_finding_pwndoc.to_pwndoc_dict()

    size = get_size(single_finding_pwndoc_dict)
    if size >= 0.3 * config.PWNDOC_REQUEST_SIZE:
        erase_msg = MdAndHTMLConvertor.html_to_pwndoc_html('<i>Erased by Importer to fit under the PwnDoc audit size limit.</i>')
        single_finding_pwndoc.set_custom_field("original_description_from_scan", erase_msg)
        single_finding_pwndoc.set_custom_field("original_description_from_template", erase_msg)
        ps.add_msg('warning', f'Finding {single_finding.fid} is too large ({round(size/1000/1000, 2)} MB). In attempt to upload it the following two fields were erased - original_description_from_scan, original_description_from_template.')
        single_finding_pwndoc_dict = single_finding_pwndoc.to_pwndoc_dict()

    single_raw_finding, changed = combine_with_existing_finding(ps, single_finding_pwndoc_dict, existing_raw_findings)
    if not changed:
        return False  # This finding is already in the Audit and the scope didn't get bigger

    try:
        upsert_raw_finding(ps.audit_id, single_raw_finding)
    except Exception as e:
        fid = single_finding.fid
        error_msg = f"Upserting finding {fid} failed. Skipping it."

        len_of_content = len(json.dumps(single_raw_finding))
        if len_of_content >= config.PWNDOC_REQUEST_SIZE * 0.95:
            error_msg += f"""\n
                (The finding might be too large.
                The limit is {config.PWNDOC_REQUEST_SIZE/1000/1000} MB while the request was {len_of_content/1000/1000} MB.)
                You can still inspect it here in Importer in the listing of findings.
            """
        logger.exception(error_msg)
        ps.add_msg('error', error_msg)
        return False
    return True


def upload_open_ports_to_pwndoc(ps: ProcessingSettings):
    meta_data = ps.load_meta_file()
    services_as_pwndoc_scope = _extend_audit_services(ps.audit_id, meta_data.get("services", {}))
    try:
        helpers.table_of_services.set_tns_table_of_services(ps.audit_id, services_as_pwndoc_scope)
    except AssertionError as e:
        if "Size must be between" in str(e):
            ps.add_msg('error', f"The table of opened ports couldn't be updated as the size limit of the audit (16MB) was reached.")
        else:
            ps.add_msg('error', f"Unknown error with table of opened ports.")


@bp.route("/meta_output/<folder_name>", methods=["GET"])
def show_meta_output(folder_name: str):
    ps: Optional[ProcessingSettings] = ProcessingSettings.load_from_folder_name(folder_name)
    if ps is None:
        return basic_msg("Folder doesn't exist"), 400

    missing_templates_splitted = [x.split("\t") for x in ps.missing_templates]

    for level, msg in ps.user_msgs:
        FlashLog.flash_loguru_msg(msg, loguru_level=level)

    if config.RQ_DISABLED is False:
        if ps.finished is False and ps.rq_is_still_in_enqueued() is False:
            FlashLog.flash_loguru_msg(
                "⚠ The processing is not marked as finished, but the background job is no longer running. It might have failed silently.",
                "warning")

    return render_template("findings_meta_output.html",
                           ps=ps,
                           missing_templates=missing_templates_splitted,
                           steps=ps.get_statuses_for_display(),
                           # import_id=folder_name,
                           )


@bp.route("/file_output/<folder_name>", methods=["GET"])
def show_file_output(folder_name: str):
    ps: Optional[ProcessingSettings] = ProcessingSettings.load_from_folder_name(folder_name)
    if ps is None:
        return basic_msg("Folder doesn't exist"), 400
    if ps.get_result_path().endswith(".docx"):  # todo: don't match on extension
        return send_file(ps.get_result_path(), as_attachment=True)

    with open(ps.get_result_path(), "r", encoding="utf8") as f:
        text = f.read()
    text = html.escape(text, quote=False)
    text = MdAndHTMLConvertor.unescape_custom_html(text)

    filename = os.path.basename(ps.get_result_path())

    return send_file(io.BytesIO(text.encode("utf8")), as_attachment=True, download_name=filename)


@bp.route("/list_findings/<folder_name>", methods=["GET"])
def list_findings(folder_name: str):
    ps: Optional[ProcessingSettings] = ProcessingSettings.load_from_folder_name(folder_name)
    if ps is None:
        return basic_msg("Folder doesn't exist"), 400
    if not ps.get_result_path().endswith(".json"):  # todo: don't match on extension
        return basic_msg("Only finding outputs to JSON can be visualized in Importer"), 400

    with open(ps.get_result_path(), "r", encoding="utf8") as f:
        data = json.load(f)

    return render_template("findings_list.html", findings=data, folder_name=folder_name)


@bp.route("/show_finding/<folder_name>/<int:finding_index>", methods=["GET"])
def show_finding(folder_name: str, finding_index: int):
    ps: Optional[ProcessingSettings] = ProcessingSettings.load_from_folder_name(folder_name)
    if ps is None:
        return basic_msg("Folder doesn't exist"), 400
    if not ps.get_result_path().endswith(".json"):  # todo: don't match on extension
        return basic_msg("Only finding outputs to JSON can be visualized in Importer"), 400

    with open(ps.get_result_path(), "r", encoding="utf8") as f:
        data = json.load(f)

    outputSafely = str_bool_to_bool(request.args.get('outputSafely', "false"))
    finding = data[finding_index]

    for attr_name, attr_value in finding.items():
        if isinstance(attr_value, str):  # todo: check if attr name is HTML field
            finding[attr_name] = MdAndHTMLConvertor.unescape_custom_html(attr_value)

    return render_template("findings_single.html", finding=finding, outputSafely=outputSafely)


def full_processing_background_wrapper(folder_name: str):
    from app import create_app
    with create_app().app_context():
        full_processing_native(folder_name)


def full_processing_native(folder_name: str):
    if not process_scanner_result_using_scan2report(folder_name):
        return

    if ProcessingSettings.load_from_folder_name(folder_name).upload_to_pwndoc:
        if not upload_scan_findings_to_pwndoc(folder_name):
            return

    ps = ProcessingSettings.load_from_folder_name(folder_name)
    ps.finished = True

    ps.fail_remaining_steps_if_finished()
    ps.save()
