Skip to content
This repository was archived by the owner on Nov 2, 2024. It is now read-only.

Commit 599f04a

Browse files
g4zeg4zemlodic
authored and
vaclavbartos
committed
crt_sh Passive_DNS playbook and visualizer (intelowlproject#2374) * created 'passive_dns' playbook and visualizer * dnsdb * validin * changes * refactor * changes * refactor + tests * changes * changes Add create user docs (intelowlproject#2381) * docs for test user creation docs for test user creation * typo :"( --------- Co-authored-by: g4ze <bhaiyajionline@gmail.com> fixed capesandbox short analysis time limit (intelowlproject#2364) * fixed capesandbox short analysis time limit * added url to soft time limit error * fixed code doctor * added update method added info installation process Orkl_search analyzer, closes intelowlproject#1274 (intelowlproject#2380) * orkl search * docs * migrations * free to use * typo --------- Co-authored-by: g4ze <bhaiyajionline@gmail.com> Frontend - no more required analyzer in scan form (intelowlproject#2397) * no more requried analyzer in scan form * fix test docs, migrations and corrections ci Co-authored-by: g4ze <bhaiyajionline@gmail.com> Co-authored-by: Matteo Lodi <30625432+mlodic@users.noreply.github.com>
1 parent c4553c8 commit 599f04a

File tree

5 files changed

+242
-0
lines changed

5 files changed

+242
-0
lines changed

‎api_app/analyzers_manager/migrations/0098_analyzer_config_crt_sh.py

+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
from django.db import migrations
2+
from django.db.models.fields.related_descriptors import (
3+
ForwardManyToOneDescriptor,
4+
ForwardOneToOneDescriptor,
5+
ManyToManyDescriptor,
6+
)
7+
8+
plugin = {
9+
"python_module": {
10+
"health_check_schedule": None,
11+
"update_schedule": None,
12+
"module": "crt_sh.Crt_sh",
13+
"base_path": "api_app.analyzers_manager.observable_analyzers",
14+
},
15+
"name": "Crt_sh",
16+
"description": "[Crt_sh](https://crt.sh/) lets you get certificates info about a domain.",
17+
"disabled": False,
18+
"soft_time_limit": 500,
19+
"routing_key": "default",
20+
"health_check_status": True,
21+
"type": "observable",
22+
"docker_based": False,
23+
"maximum_tlp": "AMBER",
24+
"observable_supported": ["domain"],
25+
"supported_filetypes": [],
26+
"run_hash": False,
27+
"run_hash_type": "",
28+
"not_supported_filetypes": [],
29+
"model": "analyzers_manager.AnalyzerConfig",
30+
}
31+
32+
params = []
33+
34+
values = []
35+
36+
37+
def _get_real_obj(Model, field, value):
38+
def _get_obj(Model, other_model, value):
39+
if isinstance(value, dict):
40+
real_vals = {}
41+
for key, real_val in value.items():
42+
real_vals[key] = _get_real_obj(other_model, key, real_val)
43+
value = other_model.objects.get_or_create(**real_vals)[0]
44+
# it is just the primary key serialized
45+
else:
46+
if isinstance(value, int):
47+
if Model.__name__ == "PluginConfig":
48+
value = other_model.objects.get(name=plugin["name"])
49+
else:
50+
value = other_model.objects.get(pk=value)
51+
else:
52+
value = other_model.objects.get(name=value)
53+
return value
54+
55+
if (
56+
type(getattr(Model, field))
57+
in [ForwardManyToOneDescriptor, ForwardOneToOneDescriptor]
58+
and value
59+
):
60+
other_model = getattr(Model, field).get_queryset().model
61+
value = _get_obj(Model, other_model, value)
62+
elif type(getattr(Model, field)) in [ManyToManyDescriptor] and value:
63+
other_model = getattr(Model, field).rel.model
64+
value = [_get_obj(Model, other_model, val) for val in value]
65+
return value
66+
67+
68+
def _create_object(Model, data):
69+
mtm, no_mtm = {}, {}
70+
for field, value in data.items():
71+
value = _get_real_obj(Model, field, value)
72+
if type(getattr(Model, field)) is ManyToManyDescriptor:
73+
mtm[field] = value
74+
else:
75+
no_mtm[field] = value
76+
try:
77+
o = Model.objects.get(**no_mtm)
78+
except Model.DoesNotExist:
79+
o = Model(**no_mtm)
80+
o.full_clean()
81+
o.save()
82+
for field, value in mtm.items():
83+
attribute = getattr(o, field)
84+
if value is not None:
85+
attribute.set(value)
86+
return False
87+
return True
88+
89+
90+
def migrate(apps, schema_editor):
91+
Parameter = apps.get_model("api_app", "Parameter")
92+
PluginConfig = apps.get_model("api_app", "PluginConfig")
93+
python_path = plugin.pop("model")
94+
Model = apps.get_model(*python_path.split("."))
95+
if not Model.objects.filter(name=plugin["name"]).exists():
96+
exists = _create_object(Model, plugin)
97+
if not exists:
98+
for param in params:
99+
_create_object(Parameter, param)
100+
for value in values:
101+
_create_object(PluginConfig, value)
102+
103+
104+
def reverse_migrate(apps, schema_editor):
105+
python_path = plugin.pop("model")
106+
Model = apps.get_model(*python_path.split("."))
107+
Model.objects.get(name=plugin["name"]).delete()
108+
109+
110+
class Migration(migrations.Migration):
111+
atomic = False
112+
dependencies = [
113+
("api_app", "0062_alter_parameter_python_module"),
114+
("analyzers_manager", "0097_analyzer_config_orklsearch"),
115+
]
116+
117+
operations = [migrations.RunPython(migrate, reverse_migrate)]

‎api_app/analyzers_manager/observable_analyzers/crt_sh.py

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import logging
2+
3+
import requests
4+
5+
from api_app.analyzers_manager import classes
6+
from tests.mock_utils import MockUpResponse, if_mock_connections, patch
7+
8+
logger = logging.getLogger(__name__)
9+
10+
11+
class Crt_sh(classes.ObservableAnalyzer):
12+
"""
13+
Wrapper of crt.sh
14+
"""
15+
16+
url = "https://crt.sh"
17+
18+
def update(self):
19+
pass
20+
21+
def run(self):
22+
headers = {"accept": "application/json"}
23+
response = requests.get(
24+
f"{self.url}/?q={self.observable_name}", headers=headers
25+
)
26+
response.raise_for_status()
27+
response = response.json()
28+
return response
29+
30+
@classmethod
31+
def _monkeypatch(cls):
32+
patches = [
33+
if_mock_connections(
34+
patch(
35+
"requests.get",
36+
return_value=MockUpResponse(
37+
{
38+
"issuer_ca_id": 16418,
39+
"issuer_name": """C=US, O=Let's Encrypt,
40+
CN=Let's Encrypt Authority X3""",
41+
"name_value": "hatch.uber.com",
42+
"min_cert_id": 325717795,
43+
"min_entry_timestamp": "2018-02-08T16:47:39.089",
44+
"not_before": "2018-02-08T15:47:39",
45+
},
46+
200,
47+
),
48+
),
49+
)
50+
]
51+
return super()._monkeypatch(patches=patches)

‎api_app/playbooks_manager/migrations/0046_add_orkl_to_free_to_use.py

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
2+
# See the file 'LICENSE' for copying permission.
3+
4+
5+
from django.db import migrations
6+
7+
8+
def migrate(apps, schema_editor):
9+
playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
10+
AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
11+
12+
pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
13+
pc.analyzers.add(AnalyzerConfig.objects.get(name="OrklSearch").id)
14+
pc.full_clean()
15+
pc.save()
16+
17+
18+
def reverse_migrate(apps, schema_editor):
19+
playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
20+
AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
21+
22+
pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
23+
pc.analyzers.remove(AnalyzerConfig.objects.get(name="OrklSearch").id)
24+
pc.full_clean()
25+
pc.save()
26+
27+
28+
class Migration(migrations.Migration):
29+
dependencies = [
30+
(
31+
"playbooks_manager",
32+
"0045_playbook_config_passive_dns",
33+
),
34+
("analyzers_manager", "0097_analyzer_config_orklsearch"),
35+
]
36+
37+
operations = [
38+
migrations.RunPython(migrate, reverse_migrate),
39+
]

‎api_app/playbooks_manager/migrations/0047_add_crt_sh_to_free_to_use.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
2+
# See the file 'LICENSE' for copying permission.
3+
4+
5+
from django.db import migrations
6+
7+
8+
def migrate(apps, schema_editor):
9+
playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
10+
AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
11+
pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
12+
pc.analyzers.add(AnalyzerConfig.objects.get(name="Crt_sh").id)
13+
pc.full_clean()
14+
pc.save()
15+
16+
17+
def reverse_migrate(apps, schema_editor):
18+
playbook_config = apps.get_model("playbooks_manager", "PlaybookConfig")
19+
AnalyzerConfig = apps.get_model("analyzers_manager", "AnalyzerConfig")
20+
pc = playbook_config.objects.get(name="FREE_TO_USE_ANALYZERS")
21+
pc.analyzers.remove(AnalyzerConfig.objects.get(name="Crt_sh").id)
22+
pc.full_clean()
23+
pc.save()
24+
25+
26+
class Migration(migrations.Migration):
27+
dependencies = [
28+
("playbooks_manager", "0046_add_orkl_to_free_to_use"),
29+
("analyzers_manager", "0098_analyzer_config_crt_sh"),
30+
]
31+
32+
operations = [
33+
migrations.RunPython(migrate, reverse_migrate),
34+
]

‎docs/source/Usage.md

+1
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ The following is the list of the available analyzers you can run out-of-the-box.
262262
* `AILTypoSquatting`:[AILTypoSquatting](https://github.com/typosquatter/ail-typo-squatting) is a Python library to generate list of potential typo squatting domains with domain name permutation engine to feed AIL and other systems.
263263
* `MalprobSearch`:[Malprob](https://malprob.io/) is a leading malware detection and identification service, powered by cutting-edge AI technology.
264264
* `OrklSearch`:[Orkl](https://orkl.eu/) is the Community Driven Cyber Threat Intelligence Library.
265+
* `Crt_sh`:[Crt_Sh](https://crt.sh/) lets you get certificates info about a domain.
265266

266267
##### Generic analyzers (email, phone number, etc.; anything really)
267268

0 commit comments

Comments
 (0)