Scan2Report ↔ Pwndoc
Choose an item from the menu above
{% endblock %}pax_global_header 0000666 0000000 0000000 00000000064 14301514725 0014514 g ustar 00root root 0000000 0000000 52 comment=2adcd41cc90f44af27dc3e00ad32697dfc735d1f
pwndocimportautomator-release-2022-08-24/ 0000775 0000000 0000000 00000000000 14301514725 0020212 5 ustar 00root root 0000000 0000000 pwndocimportautomator-release-2022-08-24/.dockerignore 0000664 0000000 0000000 00000000164 14301514725 0022667 0 ustar 00root root 0000000 0000000 .git/
.idea/
.venv/
venv-windows/
.vagrant/
__pycache__/
pwndoc/
user_data/
backups/
.env
docker.env
pwndocimportautomator-release-2022-08-24/.env.dist 0000664 0000000 0000000 00000000616 14301514725 0021750 0 ustar 00root root 0000000 0000000 # Copy this to .env for local deployment or docker.env for docker usage.
# Fill in the copied file.
PWNDOC_USERNAME=FILL_ME_IN
PWNDOC_PASSWORD=FILL_ME_IN
PWNDOC_URL=https://FILL_ME_IN/
PWNDOC_DISABLE_HTTPS_VERIFICATION=False
# For docker deployment uncomment the following two lines.
# PWNDOC_URL=https://pwndoc-backend:4242/
# PWNDOC_DISABLE_HTTPS_VERIFICATION=True
FLASK_SECRET_KEY=FILL_ME_IN
pwndocimportautomator-release-2022-08-24/.gitignore 0000664 0000000 0000000 00000000630 14301514725 0022201 0 ustar 00root root 0000000 0000000 *.json
debug_tmp/*
unified_templates/*
!/**/.gitkeep
.env
docker.env
.venv/
venv-windows
*.pyc
.idea/
.vagrant/
backups/*/*
!user_data/pwndoc-init/*
!user_data/pwndoc-config/*
user_data/mongo-data
user_data/pwndoc-docx-templates/*
user_data/importer-logs/*.log
user_data/importer-logs/*.log.gz
user_data/pwndoc-logs/*.log
test.docx
user_data/pwndoc-init/new_id_mapping.json
user_data/importer-db/main.db
pwndocimportautomator-release-2022-08-24/.gitmodules 0000664 0000000 0000000 00000000354 14301514725 0022371 0 ustar 00root root 0000000 0000000 [submodule "scan2report"]
path = scan2report
url = git@gitlab.fi.muni.cz:cybersec/tns/scan2report.git
branch = master
[submodule "pwndoc"]
path = pwndoc
url = https://github.com/BorysekOndrej/pwndoc.git
branch = customization-poc
pwndocimportautomator-release-2022-08-24/Dockerfile 0000664 0000000 0000000 00000001420 14301514725 0022201 0 ustar 00root root 0000000 0000000 FROM python:3.8
WORKDIR /app
RUN apt-get clean && apt-get update && apt-get install -y locales locales-all
RUN locale-gen cs_CZ && locale-gen cs_CZ.UTF-8 && update-locale cs_CZ.UTF-8
ENV LANG="cs_CZ.UTF-8" LC_ALL="cs_CZ.UTF-8" LC_CTYPE="cs_CZ.UTF-8"
COPY requirements.txt requirements.txt
RUN pip3 install -r requirements.txt
COPY . .
# todo: lower the timeout back to 30 seconds when the scan2report processing is done asynchronously
# 90 seconds is timeout in Firefox
ENV FLASK_APP app.py
CMD [ "gunicorn", \
"--bind", "0.0.0.0:5000", \
"-w", "4", \
"--worker-class", "gevent", \
"--timeout", "85", \
"--access-logfile", "user_data/importer-logs/gunicorn_access.log", \
"--error-logfile", "user_data/importer-logs/gunicorn_error.log", \
"app:app"]
pwndocimportautomator-release-2022-08-24/README.md 0000664 0000000 0000000 00000002124 14301514725 0021470 0 ustar 00root root 0000000 0000000 # pwndocImportAutomator
To clone repo with submodules:
```sh
git clone --recurse-submodules git@gitlab.fi.muni.cz:cybersec/tns/pwndocimportautomator.git
cd pwndocimportautomator
```
## Deployment - Docker-compose
The recommended deployment process is using Docker-compose.
```sh
# MANUAL STEP: Fill in docker.env file (use .env.dist as a template)
docker-compose up --build -d
```
How to stop the containers:
```
docker-compose down --remove-orphans
# note: flag -v would also remove the docker volume with the database (DANGEROUS)
```
Ports:
- 5001: ImportAutomator (HTTP)
- 8443: Pwndoc (HTTPS)
## First run setup steps
- Import Scan2Report templates using the web interface (they are no longer autoimported from ./scan2report/plugins/)
## How to backup
Before first run:
`sudo apt update && sudo apt install zip`
To run backup:
`./backup.sh`
## How to update
```sh
./backup.sh
git pull --recurse-submodules -X ours
./backup.sh
docker-compose up --build -d
```
## How to export scan2report plugins folder
HTTP request to `:5001/templates/export_as_scan2report_native` returns it as zip
pwndocimportautomator-release-2022-08-24/Vagrantfile 0000664 0000000 0000000 00000001557 14301514725 0022407 0 ustar 00root root 0000000 0000000 Vagrant.configure("2") do |config|
# config.vm.box = "ubuntu/xenial64"
config.vm.box = "munikypo/xubuntu-18.04"
config.vm.box_version = "0.2.0"
#map ports from local PC, to VM ... which intern will be mapped to container ports in docker-compose.yml
config.vm.network "forwarded_port",
guest: 5001, host: 9001
config.vm.network "forwarded_port",
guest: 8443, host: 9443
#allow access on windows to VM
config.vm.network "public_network", bridge: "Intel(R) Dual Band Wireless-AC 7265"
# Mount this folder as RO in the guest, since it contains secure stuff
config.vm.synced_folder ".", "/vagrant" #, :mount_options => ["ro"]
#To install, rebuild and run docker-compose on vagrant up
config.vm.provision :docker
config.vm.provision :docker_compose, yml: "/vagrant/docker-compose.yml", rebuild: true, run: "always"
end
pwndocimportautomator-release-2022-08-24/api_debug.py 0000664 0000000 0000000 00000020011 14301514725 0022475 0 ustar 00root root 0000000 0000000 from typing import List, Tuple
import time
from tqdm import tqdm
from flask import Blueprint, render_template, Response, send_file, url_for
from helpers.pwndoc_log_beautify import log_to_beautified_errors
from helpers.template_grouping import TemplateAliasing, TemplateGrouping
from helpers.custom_logging import FlashLog
import pwndoc_db_init
from helpers.file_utils import zip_folder_non_recursive_to_virtual, relative_path
from template_manager import PwndocTemplateManager
import template_pwndoc
from pwndoc_api import login, refresh_examples, delete_all_templates_in_pwndoc, load_audit_file, \
get_findings_from_audit, download_and_return_finding, delete_finding, get_audits
from helpers.db_models import DbTemplate
from template_pwndoc import TNSTemplatePwndoc, get_fid_from_raw_finding
from template_scan2report import TemplateScan2Report
from template_converters import PwndocConverter
bp = Blueprint('api_debug', __name__, template_folder='templates')
@bp.route("/", methods=["GET"])
def show_debug_actions_list():
audits_dict = get_audits()
return render_template('debug_actions.html', audits_dict=audits_dict)
@bp.route("/download_api", methods=["GET"])
def download_api_examples():
login()
refresh_examples()
return "API examples downloaded"
@bp.route("/force_upload_init_data", methods=["GET"])
def force_upload_init_data():
pwndoc_db_init.InitialData(force=True)
return "API inited"
@bp.route("/refresh_api_examples", methods=["GET"])
def debug_refresh_api_examples():
login()
refresh_examples()
return f"OK"
@bp.route("/download_logs/ and ", "").replace(" in
text = text.replace('
" + \
"FID - alias or GID - PwnDoC ID
" + \
"
".join([" - ".join(x) for x in newly_grouped_or_aliased_findings]) + \
f'
Delete all these findings (DANGEROUS; no confirmation dialog)
'
@bp.route("/delete_newly_grouped_or_aliased_findings/
" + \
"FID - alias or GID - PwnDoC ID
" + \
"
".join([" - ".join(x) for x in newly_grouped_or_aliased_findings])
@bp.route("/download_audit/
".join(answer))
@bp.route("/process_scanner_result_using_scan2report/
")
new_scope = single_raw_finding.get("scope", "").split("
")
merged_scope = original_scope + new_scope
merged_scope_deduplicated = list(dict.fromkeys(merged_scope))
fid_mapping[fid]["scope"] = "
".join(merged_scope_deduplicated)
if len(merged_scope_deduplicated) != len(new_scope):
fid_mapping[fid]["poc"] = fid_mapping[fid].get("poc", "") + "
=====
" + single_raw_finding.get("poc", "")
single_raw_finding["_changed"] = True
new_or_changed = list(filter(lambda x: x.get("_changed", True), fid_mapping.values()))
logger.info(f"Import contained {len(list(fid_mapping.values()))} unique FIDs which resulted in upsert of {len(new_or_changed)} findings.")
return new_or_changed
def _extend_audit_services(audit_id: str, new_services: dict) -> helpers.table_of_services.Scope:
answer = helpers.table_of_services.get_tns_table_of_services(audit_id)
persistent_scopes_file_path = persistent_audit_scopes_file_path(audit_id)
local_services_copy = json_safe_load(persistent_scopes_file_path)
if local_services_copy is not None:
for host in local_services_copy.get('scope', {}).get('hosts', []):
for service in host.get('services', []):
answer.add_entry(host['ip'], service['port'], service['protocol'], service['name'])
for host_ip, port_list in new_services.items():
for port_number, l3_protocol_list in port_list.items():
for l3_protocol, app_protocol in l3_protocol_list.items():
answer.add_entry(host_ip, port_number, l3_protocol, app_protocol)
return answer
@bp.route("/force_upload_services/
and (un)escaping is handled separately
@staticmethod
def _get_basic_html_tags():
simple_tags = ['b', 'i', 'br', 'p', 'ul', 'li', 'strong', 's', 'u', 'p', 'code', 'strong', 'em', 'div', 'span']
answer = []
for tag in simple_tags:
answer.append(f'<{tag}>')
answer.append(f'{tag}>')
return answer
# BEGIN: MD -> HTML
@staticmethod
def __is_md_separator(char: str) -> bool:
# warning: This is not up to spec, but slightly more involved than scan2report version.
# correct spec would be: https://spec.commonmark.org/0.30/#emphasis-and-strong-emphasis
return char.isspace() or char.isnumeric() or char in '''!"#$%&'()+, -./:;<=>?@[]^`{|}~'''
@classmethod
def __is_valid_md_state_change(cls, current_state: bool, char_before: str, char_after: str):
# warning: This is not up to spec.
# if (not current_state and cls.__is_md_separator(char_before) and not cls.__is_md_separator(char_after)) or \
# (current_state and not cls.__is_md_separator(char_before) and cls.__is_md_separator(char_after)):
return True
@classmethod
def __replace_simple_tag_with_pair_tag(cls, text: str, original_tag: str, opening_tag: str, closing_tag: str) -> str:
answer = ""
remaining_string = text
current_state = False
while True:
pre, sep, post = remaining_string.partition(original_tag)
answer += pre
if len(sep) == 0:
break
char_before = (" " + pre)[-1]
char_after = (post + " ")[0]
if cls.__is_valid_md_state_change(current_state, char_before, char_after):
current_state = not current_state
answer += opening_tag if current_state else closing_tag
else:
answer += sep
remaining_string = post
if current_state:
logger.warning(f"Conversion found possible unmatched tag {original_tag}")
return answer
@classmethod
def _replace_md_pair_tags(cls, text: str) -> str:
for md_tag, html_tags in cls._MD_TO_HTML_PAIR_TAGS.items():
text = cls.__replace_simple_tag_with_pair_tag(text, md_tag, html_tags[0], html_tags[1])
return text
@classmethod
def reverse_custom_escaping_of_html(cls, text: str) -> str:
for html_char, custom_escaped_char in cls._HTML_TO_RESERVED_CHARS.items():
text = text.replace(custom_escaped_char, html_char)
unicode_repr = ascii(custom_escaped_char).strip("'")
text = text.replace(unicode_repr, html_char) # Handles unicode repr (e.g. \ue020) which happens for example when outputting to JSON.
return text
# BEGIN: HTML -> MD
@classmethod
def _normalize_html_tags(cls, text: str) -> str:
"""Normalize (partially) the HTML tags (i.e. -> )"""
for original_tag, new_tag in cls._HTML_TAG_NORMALIZATION.items():
text = text.replace(original_tag, new_tag)
return text
@classmethod
def _replace_html_pair_tags_with_md_equivalents(cls, text: str) -> str:
""" Convert HTML formatting to MD (i.e. -> **) """
for md_tag, html_tags in cls._MD_TO_HTML_PAIR_TAGS.items():
for html_tag in html_tags:
text = text.replace(html_tag, md_tag)
return text
@classmethod
def _strip_unsupported_html_tags(cls, text: str) -> str:
""" Strip (some) unsupported HTML tags (i.e.
) """
for html_tag, md_tag in cls._HTML_TAGS_TO_STRIP_FOR_RAW_MD.items():
text = text.replace(f"<{html_tag}>", md_tag)
text = text.replace(f"{html_tag}>", md_tag)
return text
@classmethod
def _custom_escape_html(cls, text: str) -> str:
""" Escapes HTML special chars so that it's possible to distinguish them from any HTML chars added later in the process by scan2report. """
for html_char, custom_escaped_char in cls._HTML_TO_RESERVED_CHARS.items():
text = text.replace(html_char, custom_escaped_char)
return text
# BEGIN: Public interface
@classmethod
def md_to_html(cls, md_text: str) -> str:
answer = md_text
answer = html.escape(answer)
answer = cls._replace_md_pair_tags(answer)
answer = cls.reverse_custom_escaping_of_html(answer)
answer = answer.replace("\n", "
")
return answer
@classmethod
def html_to_md(cls, html_text: str, output_without_html_tags: bool = False) -> str:
answer = html_text
answer = cls._normalize_html_tags(answer)
answer = cls.html_to_pwndoc_html(answer)
if output_without_html_tags:
answer = answer.replace("
", "\n")
answer = cls._replace_html_pair_tags_with_md_equivalents(answer)
answer = cls._strip_unsupported_html_tags(answer)
answer = html.unescape(answer)
else:
answer = cls._custom_escape_html(answer)
return answer
@classmethod
def force_unescape_basic_tags(cls, text: str) -> str:
simple_tags = cls._get_basic_html_tags()
for full_tag_html in simple_tags:
full_tag_escaped = html.escape(full_tag_html)
text = text.replace(full_tag_escaped, full_tag_html)
# note: this only covers first level encoded, i.e. <p> but not <p>
# it also does not cover custom encoding, however neither of those should occur
return text
@staticmethod
def html_to_pwndoc_html(text: str) -> str:
text = html.unescape(text)
# PwnDoc is extremely strict in regards to HTML standard.
text = text.replace("
") # HTML doesn't allow ', '
') # HTML doesn't allow
text = f'
{text}
' # PwnDoc requires all text to be in some of the allowed tags. bs = BeautifulSoup(text, 'html.parser') # PwnDoc doesn't support tags yet. links = [] for a in bs.find_all('a'): content = a.string url = a.get('href') if url is None: a.replace_with(content) else: links.append(url) a.replace_with(f"{content} [{len(links)}]") # HTML doesn't allow\n"
for i, link in enumerate(links):
text += f"[{i+1}]: {link}
\n"
text += "
Choose an item from the menu above
{% endblock %}{% endblock %} pwndocimportautomator-release-2022-08-24/templates/navbar.html 0000664 0000000 0000000 00000004533 14301514725 0024354 0 ustar 00root root 0000000 0000000 pwndocimportautomator-release-2022-08-24/templates/upload_scanner_result_get.html 0000664 0000000 0000000 00000005001 14301514725 0030324 0 ustar 00root root 0000000 0000000 {% extends "base.html" %} {% block title %}Process results from scanners{% endblock %} {% block content %}