Skip to content

Commit a0c0ce5

Browse files
sebixsebix
authored
Merge pull request #2338 from certtools/shadowserver-20230315
Add 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Ser…
2 parents 872154d + 746a0f0 commit a0c0ce5

File tree

5 files changed

+169
-3
lines changed

5 files changed

+169
-3
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ CHANGELOG
2020
#### Collectors
2121

2222
#### Parsers
23+
- `intelmq.bots.parsers.shadowserver._config`:
24+
- Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338 by elsif2)
25+
- Removed unused 'p0f_genre' and 'p0f_detail' from the 'DNS-Open-Resolvers' report. (PR#2338 by elsif2)
2326

2427
#### Experts
2528

‎intelmq/bots/parsers/shadowserver/_config.py‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1557,8 +1557,6 @@ def scan_exchange_identifier(field):
15571557
('source.geolocation.cc', 'geo'),
15581558
('source.geolocation.region', 'region'),
15591559
('source.geolocation.city', 'city'),
1560-
('os.name', 'p0f_genre'),
1561-
('os.version', 'p0f_detail'),
15621560
('extra.', 'naics', invalidate_zero),
15631561
('extra.', 'sic', invalidate_zero),
15641562
('extra.', 'sector', validate_to_none),
@@ -3392,7 +3390,7 @@ def scan_exchange_identifier(field):
33923390
},
33933391
}
33943392

3395-
# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SNMP
3393+
# https://www.shadowserver.org/what-we-do/network-reporting/open-snmp-report/
33963394
scan_snmp = {
33973395
'required_fields': [
33983396
('time.source', 'timestamp', add_UTC_to_timestamp),
@@ -4067,6 +4065,37 @@ def scan_exchange_identifier(field):
40674065
},
40684066
}
40694067

4068+
# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ws-discovery-service-report/
4069+
scan_ws_discovery = {
4070+
'required_fields': [
4071+
('time.source', 'timestamp', add_UTC_to_timestamp),
4072+
('source.ip', 'ip', validate_ip),
4073+
('source.port', 'port', convert_int),
4074+
],
4075+
'optional_fields': [
4076+
('protocol.transport', 'protocol'),
4077+
('source.reverse_dns', 'hostname'),
4078+
('extra.', 'tag', validate_to_none),
4079+
('source.asn', 'asn', invalidate_zero),
4080+
('source.geolocation.cc', 'geo'),
4081+
('source.geolocation.region', 'region'),
4082+
('source.geolocation.city', 'city'),
4083+
('extra.source.naics', 'naics', invalidate_zero),
4084+
('extra.source.sic', 'sic', invalidate_zero),
4085+
('extra.source.sector', 'sector', validate_to_none),
4086+
('extra.', 'response_size', convert_int),
4087+
('extra.', 'amplification', convert_float),
4088+
('extra.', 'error', validate_to_none),
4089+
('extra.', 'raw_response', validate_to_none),
4090+
],
4091+
'constant_fields': {
4092+
'classification.taxonomy': 'vulnerable',
4093+
'classification.type': 'vulnerable-system',
4094+
'protocol.application': 'ws-discovery',
4095+
'classification.identifier': 'open-ws-discovery',
4096+
},
4097+
}
4098+
40704099
# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/
40714100
scan_xdmcp = {
40724101
'required_fields': [
@@ -4116,6 +4145,13 @@ def scan_exchange_identifier(field):
41164145
('extra.', 'source', validate_to_none),
41174146
('extra.', 'sender', validate_to_none),
41184147
('extra.', 'subject', validate_to_none),
4148+
('source.ip', 'src_ip', validate_ip),
4149+
('source.asn', 'src_asn', invalidate_zero),
4150+
('source.geolocation.cc', 'src_geo'),
4151+
('source.geolocation.region', 'src_region'),
4152+
('source.geolocation.city', 'src_city'),
4153+
('extra.source.naics', 'src_naics', invalidate_zero),
4154+
('extra.source.sector', 'src_sector', validate_to_none),
41194155
('malware.hash.md5', 'md5', validate_to_none),
41204156
],
41214157
'constant_fields': {
@@ -4186,6 +4222,7 @@ def scan_exchange_identifier(field):
41864222
('Sandbox-URL', 'sandbox_url', sandbox_url),
41874223
('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp),
41884224
('IPv6-DNS-Open-Resolvers', 'scan6_dns', scan_dns),
4225+
('IPv6-Vulnerable-Exchange', 'scan6_exchange', scan_exchange),
41894226
('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp),
41904227
('IPv6-Accessible-HTTP', 'scan6_http', scan_http),
41914228
('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable),
@@ -4277,6 +4314,7 @@ def scan_exchange_identifier(field):
42774314
('Open-TFTP', 'scan_tftp', scan_tftp),
42784315
('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti),
42794316
('Accessible-VNC', 'scan_vnc', scan_vnc),
4317+
('Accessible-WS-Discovery-Service', 'scan_ws_discovery', scan_ws_discovery),
42804318
('Open-XDMCP', 'scan_xdmcp', scan_xdmcp),
42814319
('Spam-URL', 'spam_url', spam_url),
42824320
('Special', 'special', special),

‎intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# SPDX-FileCopyrightText: 2022 Shadowserver Foundation
2+
#
3+
# SPDX-License-Identifier: AGPL-3.0-or-later
4+
5+
# -*- coding: utf-8 -*-
6+
7+
import os
8+
import unittest
9+
10+
import intelmq.lib.test as test
11+
import intelmq.lib.utils as utils
12+
from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot
13+
14+
with open(os.path.join(os.path.dirname(__file__),
15+
'testdata/scan_ws_discovery.csv')) as handle:
16+
EXAMPLE_FILE = handle.read()
17+
EXAMPLE_LINES = EXAMPLE_FILE.splitlines()
18+
19+
EXAMPLE_REPORT = {'feed.name': 'Accessible-WS-Discovery-Service',
20+
"raw": utils.base64_encode(EXAMPLE_FILE),
21+
"__type": "Report",
22+
"time.observation": "2010-02-10T00:00:00+00:00",
23+
"extra.file_name": "2010-02-10-scan_ws_discovery-test.csv",
24+
}
25+
EVENTS = [
26+
{
27+
'__type': 'Event',
28+
'classification.identifier': 'open-ws-discovery',
29+
'classification.taxonomy': 'vulnerable',
30+
'classification.type': 'vulnerable-system',
31+
'extra.amplification': 164.83,
32+
'extra.error': 'Validation constraint violation: SOAP message expected',
33+
'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK',
34+
'extra.response_size': 989,
35+
'extra.source.sector': 'Communications, Service Provider, and Hosting Service',
36+
'extra.tag': 'ws-discovery',
37+
'feed.name': 'Accessible-WS-Discovery-Service',
38+
'protocol.application': 'ws-discovery',
39+
'protocol.transport': 'udp',
40+
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])),
41+
'source.asn': 64512,
42+
'source.geolocation.cc': 'ZZ',
43+
'source.geolocation.city': 'City',
44+
'source.geolocation.region': 'Region',
45+
'source.ip': '192.168.0.1',
46+
'source.port': 3702,
47+
'source.reverse_dns': 'node01.example.com',
48+
'time.source': '2010-02-10T00:00:00+00:00'
49+
},
50+
{
51+
'__type': 'Event',
52+
'classification.identifier': 'open-ws-discovery',
53+
'classification.taxonomy': 'vulnerable',
54+
'classification.type': 'vulnerable-system',
55+
'extra.amplification': 183.6,
56+
'extra.error': 'Validation constraint violation: missing root element',
57+
'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK',
58+
'extra.response_size': 918,
59+
'extra.source.sector': 'Communications, Service Provider, and Hosting Service',
60+
'extra.tag': 'ws-discovery',
61+
'feed.name': 'Accessible-WS-Discovery-Service',
62+
'protocol.application': 'ws-discovery',
63+
'protocol.transport': 'udp',
64+
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])),
65+
'source.asn': 64512,
66+
'source.geolocation.cc': 'ZZ',
67+
'source.geolocation.city': 'City',
68+
'source.geolocation.region': 'Region',
69+
'source.ip': '192.168.0.2',
70+
'source.port': 3702,
71+
'source.reverse_dns': 'node02.example.com',
72+
'time.source': '2010-02-10T00:00:01+00:00'
73+
},
74+
{
75+
'__type': 'Event',
76+
'classification.identifier': 'open-ws-discovery',
77+
'classification.taxonomy': 'vulnerable',
78+
'classification.type': 'vulnerable-system',
79+
'extra.amplification': 197.8,
80+
'extra.error': 'Validation constraint violation: SOAP message expected',
81+
'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK',
82+
'extra.response_size': 989,
83+
'extra.source.sector': 'Communications, Service Provider, and Hosting Service',
84+
'extra.tag': 'ws-discovery',
85+
'feed.name': 'Accessible-WS-Discovery-Service',
86+
'protocol.application': 'ws-discovery',
87+
'protocol.transport': 'udp',
88+
'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])),
89+
'source.asn': 64512,
90+
'source.geolocation.cc': 'ZZ',
91+
'source.geolocation.city': 'City',
92+
'source.geolocation.region': 'Region',
93+
'source.ip': '192.168.0.3',
94+
'source.port': 3702,
95+
'source.reverse_dns': 'node03.example.com',
96+
'time.source': '2010-02-10T00:00:02+00:00'
97+
}
98+
]
99+
100+
101+
class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase):
102+
"""
103+
A TestCase for a ShadowserverParserBot.
104+
"""
105+
106+
@classmethod
107+
def set_bot(cls):
108+
cls.bot_reference = ShadowserverParserBot
109+
cls.default_input_message = EXAMPLE_REPORT
110+
111+
def test_event(self):
112+
""" Test if correct Event has been produced. """
113+
self.run_bot()
114+
for i, EVENT in enumerate(EVENTS):
115+
self.assertMessageEqual(i, EVENT)
116+
117+
118+
if __name__ == '__main__': # pragma: no cover
119+
unittest.main()

‎intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","response_size","amplification","error","raw_response"
2+
"2010-02-10 00:00:00",192.168.0.1,udp,3702,node01.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,164.83,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK
3+
"2010-02-10 00:00:01",192.168.0.2,udp,3702,node02.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",918,183.60,"Validation constraint violation: missing root element",c2FtcGxlIHJlc3BvbnNlIGRhdGEK
4+
"2010-02-10 00:00:02",192.168.0.3,udp,3702,node03.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,197.80,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK

‎intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
SPDX-FileCopyrightText: 2023 The Shadowserver Foundation
2+
SPDX-License-Identifier: AGPL-3.0-or-later

0 commit comments

Comments
 (0)