1"""The `suricata_check.utils.regex` module contains regular expressions for matching various parts of rules."""
2
3import logging
4from collections.abc import Iterable, Sequence
5from functools import lru_cache
6
7from suricata_check.utils.regex_provider import Pattern
8from suricata_check.utils.regex_provider import (
9 get_regex_provider as _get_regex_provider,
10)
11from suricata_check.utils.rule import Rule
12
13_logger = logging.getLogger(__name__)
14_regex_provider = _get_regex_provider()
15
16LRU_CACHE_SIZE = 10
17
18ADDRESS_GROUPS = (
19 "HOME_NET",
20 "EXTERNAL_NET",
21 "HTTP_SERVERS",
22 "SMTP_SERVERS",
23 "SQL_SERVERS",
24 "DNS_SERVERS",
25 "TELNET_SERVERS",
26 "AIM_SERVERS",
27 "DC_SERVERS",
28 "DNP3_SERVER",
29 "DNP3_CLIENT",
30 "MODBUS_CLIENT",
31 "MODBUS_SERVER",
32 "ENIP_CLIENT",
33 "ENIP_SERVER",
34)
35
36
37PORT_GROUPS = (
38 "HTTP_PORTS",
39 "SHELLCODE_PORTS",
40 "ORACLE_PORTS",
41 "SSH_PORTS",
42 "DNP3_PORTS",
43 "MODBUS_PORTS",
44 "FILE_DATA_PORTS",
45 "FTP_PORTS",
46 "GENEVE_PORTS",
47 "VXLAN_PORTS",
48 "TEREDO_PORTS",
49)
50
51ALL_VARIABLES = ADDRESS_GROUPS + PORT_GROUPS
52
53CLASSTYPES = (
54 "not-suspicious",
55 "unknown",
56 "bad-unknown",
57 "attempted-recon",
58 "successful-recon-limited",
59 "successful-recon-largescale",
60 "attempted-dos",
61 "successful-dos",
62 "attempted-user",
63 "unsuccessful-user",
64 "successful-user",
65 "attempted-admin",
66 "successful-admin",
67 # NEW CLASSIFICATIONS
68 "rpc-portmap-decode",
69 "shellcode-detect",
70 "string-detect",
71 "suspicious-filename-detect",
72 "suspicious-login",
73 "system-call-detect",
74 "tcp-connection",
75 "trojan-activity",
76 "unusual-client-port-connection",
77 "network-scan",
78 "denial-of-service",
79 "non-standard-protocol",
80 "protocol-command-decode",
81 "web-application-activity",
82 "web-application-attack",
83 "misc-activity",
84 "misc-attack",
85 "icmp-event",
86 "inappropriate-content",
87 "policy-violation",
88 "default-login-attempt",
89 # Update
90 "targeted-activity",
91 "exploit-kit",
92 "external-ip-check",
93 "domain-c2",
94 "pup-activity",
95 "credential-theft",
96 "social-engineering",
97 "coin-mining",
98 "command-and-control",
99)
100
101NON_FUNCTIONAL_KEYWORDS = (
102 "classtype",
103 "gid",
104 "metadata",
105 "msg",
106 "priority",
107 "reference",
108 "rev",
109 "sid",
110 "target",
111)
112
113FLOW_KEYWORDS = (
114 "flow",
115 "flow.age",
116 "flowint",
117)
118
119FLOW_OPTIONS: Sequence[str] = (
120 "to_client",
121 "to_server",
122 "from_client",
123 "from_server",
124 "established",
125 "not_established",
126 "stateless",
127 "only_stream",
128 "no_stream",
129 "only_frag",
130 "no_frag",
131)
132
133STREAM_KEYWORDS = ("stream_size",)
134
135FLOW_STREAM_KEYWORDS: Sequence[str] = tuple(
136 sorted(set(FLOW_KEYWORDS).union(STREAM_KEYWORDS)),
137)
138
139STICKY_BUFFER_NAMING = {
140 "dce_iface": "dce.iface",
141 "dce_opnum": "dce.opnum",
142 "dce_stub_data": "dce.stub_data",
143 "dns_query": "dns.query",
144 "file_data": "file.data",
145 "http_accept": "http.accept",
146 "http_accept_enc": "http.accept_enc",
147 "http_accept_lang": "http.accept_lang",
148 "http_client_body": "http.request_body",
149 "http_connection": "http.connection",
150 "http_content_len": "http.content_len",
151 "http_content_type": "http.content_type",
152 "http_cookie": "http.cookie",
153 "http_header": "http.header",
154 "http_header_names": "http.header_names",
155 "http_host": "http.host",
156 "http_method": "http.method",
157 "http_protocol": "http.protocol",
158 "http_raw_header": "http.header.raw",
159 "http_raw_host": "http.host.raw",
160 "http_raw_uri": "http.uri.raw",
161 "http_referer": "http.referer",
162 "http_request_line": "http.request_line",
163 "http_response_line": "http.response_line",
164 "http_server_body": "http.response_body",
165 "http_start": "http.start",
166 "http_stat_code": "http.stat_code",
167 "http_stat_msg": "http.stat_msg",
168 "http_uri": "http.uri",
169 "http_user_agent": "http.user_agent",
170 "ja3_hash": "ja3.hash",
171 "tls_cert_fingerprint": "tls.cert_fingerprint",
172 "tls_cert_issuer": "tls.cert_issuer",
173 "tls_cert_serial": "tls.cert_serial",
174 "tls_cert_subject": "tls.cert_subject",
175 "tls_sni": "tls.sni",
176}
177
178BASE64_BUFFER_KEYWORDS = ("base64_data",)
179
180OTHER_BUFFERS = (
181 "http.location",
182 "http.request_header",
183 "http.response_header",
184 "http.server",
185 "ja3s.hash",
186 "tls.certs",
187 "tls.version",
188)
189
190assert set(OTHER_BUFFERS).isdisjoint(
191 set(STICKY_BUFFER_NAMING.keys()).union(STICKY_BUFFER_NAMING.values()),
192)
193
194BUFFER_KEYWORDS: Sequence[str] = tuple(
195 sorted(
196 set(STICKY_BUFFER_NAMING.keys())
197 .union(STICKY_BUFFER_NAMING.values())
198 .union(BASE64_BUFFER_KEYWORDS)
199 .union(OTHER_BUFFERS),
200 ),
201)
202
203SIZE_KEYWORDS = (
204 "bsize",
205 "dsize",
206)
207
208TRANSFORMATION_KEYWORDS = (
209 "compress_whitespace",
210 "dotprefix",
211 "header_lowercase",
212 "pcrexform",
213 "strip_pseudo_headers",
214 "strip_whitespace",
215 "to_lowercase",
216 "to_md5",
217 "to_sha1",
218 "to_sha256",
219 "to_uppercase",
220 "url_decode",
221 "xor",
222)
223
224BASE64_TRANSFORMATION_KEYWORDS = ("base64_decode",)
225
226ALL_TRANSFORMATION_KEYWORDS: Sequence[str] = tuple(
227 sorted(set(TRANSFORMATION_KEYWORDS).union(BASE64_TRANSFORMATION_KEYWORDS)),
228)
229
230CONTENT_KEYWORDS = ("content", "pcre")
231
232POINTER_MOVEMENT_KEYWORDS = (
233 "depth",
234 "distance",
235 "offset",
236 "pkt_data",
237 "within",
238)
239
240COMPATIBILITY_MODIFIER_KEYWORDS = ("rawbytes",)
241
242MODIFIER_KEYWORDS = ("nocase",)
243
244ALL_MODIFIER_KEYWORDS: Sequence[str] = tuple(
245 sorted(set(COMPATIBILITY_MODIFIER_KEYWORDS).union(MODIFIER_KEYWORDS)),
246)
247
248MATCH_LOCATION_KEYWORDS = (
249 "endswith",
250 "startswith",
251)
252
253OTHER_PAYLOAD_KEYWORDS = (
254 "byte_extract",
255 "byte_jump",
256 "byte_test",
257 "isdataat",
258)
259
260IP_SPECIFIC_KEYWORDS = (
261 "ip_proto",
262 "ttl",
263)
264
265TCP_SPECIFIC_KEYWORDS = (
266 "ack",
267 "flags", # This is a duplicate of tcp.flags
268 "seq",
269 "tcp.flags",
270 "tcp.hdr",
271)
272
273UDP_SPECIFIC_KEYWORDS = ("udp.hdr",)
274
275ICMP_SPECIFIC_KEYWORDS = (
276 "fragbits",
277 "icode",
278 "icmp_id",
279 "icmp_seq",
280 "itype",
281)
282
283HTTP_SPECIFIC_KEYWORDS = (
284 "file.data",
285 "file_data",
286 "http.accept",
287 "http.accept_enc",
288 "http.accept_lang",
289 "http.connection",
290 "http.content_len",
291 "http.content_len",
292 "http.content_type",
293 "http.cookie",
294 "http.header",
295 "http.header_names",
296 "http.header.raw",
297 "http.host",
298 "http.host.raw",
299 "http.location",
300 "http.method",
301 "http.protocol",
302 "http.referer",
303 "http.request_body",
304 "http.request_header",
305 "http.request_line",
306 "http.response_body",
307 "http.response_header",
308 "http.response_line",
309 "http.server",
310 "http.start",
311 "http.stat_code",
312 "http.stat_code",
313 "http.stat_msg",
314 "http.uri",
315 "http.uri.raw",
316 "http.user_agent",
317 "http_accept",
318 "http_accept_enc",
319 "http_accept_lang",
320 "http_connection",
321 "http_content_len",
322 "http_content_len",
323 "http_content_type",
324 "http_cookie",
325 "http_header",
326 "http_header_names",
327 "http_host",
328 "http_location",
329 "http_method",
330 "http_protocol",
331 "http_raw_header",
332 "http_raw_host",
333 "http_raw_uri",
334 "http_referer",
335 "http_request_line",
336 "http_response_line",
337 "http_server_body",
338 "http_start",
339 "http_stat_code",
340 "http_stat_msg",
341 "http_uri",
342 "http_user_agent",
343 "urilen",
344)
345
346DNS_SPECIFIC_KEYWORDS = (
347 "dns.opcode",
348 "dns.query",
349 "dns_query",
350)
351
352TLS_SPECIFIC_KEYWORDS = (
353 "ssl_version",
354 "ssl_state",
355 "tls.cert_fingerprint",
356 "tls.cert_issuer",
357 "tls.cert_serial",
358 "tls.cert_subject",
359 "tls.certs",
360 "tls.sni",
361 "tls.version",
362 "tls_cert_fingerprint",
363 "tls_cert_issuer",
364 "tls_cert_serial",
365 "tls_cert_subject",
366 "tls_sni",
367)
368
369SSH_SPECIFIC_KEYWORDS = ("ssh_proto",)
370
371JA3_JA4_KEYWORDS = (
372 "ja3.hash",
373 "ja3_hash",
374 "ja3.string",
375 "ja3s.hash",
376)
377
378DCERPC_SPECIFIC_KEYWORDS = (
379 "dce.iface",
380 "dce.opnum",
381 "dce.stub_data",
382 "dce_iface",
383 "dce_opnum",
384 "dce_stub_data",
385)
386
387FTP_KEYWORDS = ("ftpbounce", "ftpdata_command")
388
389APP_LAYER_KEYWORDS = (
390 "app-layer-event",
391 "app-layer-protocol",
392)
393
394PROTOCOL_SPECIFIC_KEYWORDS = tuple(
395 sorted(
396 set().union(
397 *(
398 IP_SPECIFIC_KEYWORDS,
399 TCP_SPECIFIC_KEYWORDS,
400 UDP_SPECIFIC_KEYWORDS,
401 ICMP_SPECIFIC_KEYWORDS,
402 HTTP_SPECIFIC_KEYWORDS,
403 DNS_SPECIFIC_KEYWORDS,
404 TLS_SPECIFIC_KEYWORDS,
405 SSH_SPECIFIC_KEYWORDS,
406 DCERPC_SPECIFIC_KEYWORDS,
407 JA3_JA4_KEYWORDS,
408 FTP_KEYWORDS,
409 APP_LAYER_KEYWORDS,
410 ),
411 ),
412 ),
413)
414
415PERFORMANCE_DETECTION_OPTIONS = ("fast_pattern",)
416
417LUA_KEYWORDS = ("lua", "luajit")
418
419ALL_DETECTION_KEYWORDS: Sequence[str] = tuple(
420 sorted(
421 set().union(
422 *(
423 BUFFER_KEYWORDS,
424 SIZE_KEYWORDS,
425 ALL_TRANSFORMATION_KEYWORDS,
426 CONTENT_KEYWORDS,
427 POINTER_MOVEMENT_KEYWORDS,
428 ALL_MODIFIER_KEYWORDS,
429 MATCH_LOCATION_KEYWORDS,
430 OTHER_PAYLOAD_KEYWORDS,
431 PROTOCOL_SPECIFIC_KEYWORDS,
432 PERFORMANCE_DETECTION_OPTIONS,
433 LUA_KEYWORDS,
434 ),
435 ),
436 ),
437)
438
439THRESHOLD_KEYWORDS = (
440 "detection_filter",
441 "threshold",
442)
443
444STATEFUL_KEYWORDS = ("flowbits", "flowint", "xbits")
445
446OTHER_KEYWORDS = ("noalert", "tag")
447
448ALL_KEYWORDS = tuple(
449 sorted(
450 set().union(
451 *(
452 NON_FUNCTIONAL_KEYWORDS,
453 FLOW_KEYWORDS,
454 STREAM_KEYWORDS,
455 ALL_DETECTION_KEYWORDS,
456 THRESHOLD_KEYWORDS,
457 STATEFUL_KEYWORDS,
458 OTHER_KEYWORDS,
459 ),
460 ),
461 ),
462)
463
464METADATA_DATE_KEYWORDS = (
465 "created_at",
466 "reviewed_at",
467 "updated_at",
468)
469
470METADATA_NON_DATE_KEYWORDS = (
471 "affected_product",
472 "attack_target",
473 "confidence",
474 "cve",
475 "deprecation_reason",
476 "deployment",
477 "former_category",
478 "former_sid",
479 "impact_flag",
480 "malware_family",
481 "mitre_tactic_id",
482 "mitre_tactic_name",
483 "mitre_technique_id",
484 "mitre_technique_name",
485 "performance_impact",
486 "policy",
487 "ruleset",
488 "signature_severity",
489 "tag",
490 "tls_state",
491 "first_seen",
492 "confidence_level",
493)
494
495SURICATA_CHECK_METADATA_KEYWORDS = ("suricata-check",)
496
497ALL_METADATA_KEYWORDS = tuple(
498 sorted(
499 set(METADATA_DATE_KEYWORDS)
500 .union(METADATA_NON_DATE_KEYWORDS)
501 .union(SURICATA_CHECK_METADATA_KEYWORDS),
502 ),
503)
504
505IP_ADDRESS_REGEX = _regex_provider.compile(r"^.*\d+\.\d+\.\d+\.\d+.*$")
506
507_GROUP_REGEX = _regex_provider.compile(r"^(!)?\[(.*)\]$")
508_VARIABLE_GROUP_REGEX = _regex_provider.compile(r"^!?\$([A-Z\_]+)$")
509
510_ACTION_REGEX = _regex_provider.compile(
511 r"(alert|pass|drop|reject|rejectsrc|rejectdst|rejectboth)",
512)
513_PROTOCOL_REGEX = _regex_provider.compile(r"[a-z0-3\-]+")
514_ADDR_REGEX = _regex_provider.compile(r"[a-zA-Z0-9\$_\!\[\],\s/\.]+")
515_PORT_REGEX = _regex_provider.compile(r"[a-zA-Z0-9\$_\!\[\],\s:]+")
516_DIRECTION_REGEX = _regex_provider.compile(r"(\->|<>)")
517HEADER_REGEX = _regex_provider.compile(
518 rf"{_ACTION_REGEX.pattern}\s*{_PROTOCOL_REGEX.pattern}\s*{_ADDR_REGEX.pattern}\s*{_PORT_REGEX.pattern}\s*{_DIRECTION_REGEX.pattern}\s*{_ADDR_REGEX.pattern}\s*{_PORT_REGEX.pattern}",
519)
520_OPTION_REGEX = _regex_provider.compile(
521 r"[a-z\-\._]+\s*(:(\s*([0-9]+|.+)\s*\,?\s*)+)?;",
522)
523_BODY_REGEX = _regex_provider.compile(rf"\((\s*{_OPTION_REGEX.pattern}\s*)*\)")
524_RULE_REGEX = _regex_provider.compile(
525 rf"^(\s*#)?\s*{HEADER_REGEX.pattern}\s*{_BODY_REGEX.pattern}\s*(#.*)?$",
526)
527
528
529@lru_cache(maxsize=LRU_CACHE_SIZE)
530def __escape_regex(s: str) -> str:
531 # Escape the escape character first
532 s = s.replace("\\", "\\\\")
533
534 # Then escape all other characters
535 # . ^ $ * + ? { } [ ] \ | ( )
536 s = s.replace(".", "\\.")
537 s = s.replace("^", "\\^")
538 s = s.replace("$", "\\$")
539 s = s.replace("*", "\\*")
540 s = s.replace("+", "\\+")
541 s = s.replace("?", "\\?")
542 s = s.replace("{", "\\{")
543 s = s.replace("}", "\\}")
544 s = s.replace("[", "\\[")
545 s = s.replace("]", "\\]")
546 s = s.replace("|", "\\|")
547 s = s.replace("(", "\\(")
548 s = s.replace(")", "\\)")
549
550 return s # noqa: RET504
551
552
[docs]
553def get_options_regex(options: Iterable[str]) -> Pattern:
554 """Returns a regular expression that can match any of the provided options."""
555 return __get_options_regex(tuple(sorted(options)))
556
557
558@lru_cache(maxsize=LRU_CACHE_SIZE)
559def __get_options_regex(options: Sequence[str]) -> Pattern:
560 return _regex_provider.compile(
561 "(" + "|".join([__escape_regex(option) for option in options]) + ")",
562 )
563
564
565def __is_group(entry: str) -> bool:
566 if _GROUP_REGEX.match(entry) is None:
567 return False
568
569 return True
570
571
[docs]
572def get_rule_group_entries(group: str) -> Sequence[str]:
573 """Returns a list of entries in a group."""
574 stripped_group = group.strip()
575
576 if not __is_group(stripped_group):
577 return [stripped_group]
578
579 match = _GROUP_REGEX.match(stripped_group)
580 assert match is not None
581 negated = match.group(1) == "!"
582
583 entries = []
584 for entry in match.group(2).split(","):
585 stripped_entry = entry.strip()
586 if __is_group(stripped_entry):
587 entries += get_rule_group_entries(stripped_entry)
588 else:
589 entries.append(stripped_entry)
590
591 if negated:
592 entries = ["!" + entry for entry in entries]
593
594 return entries
595
596
[docs]
597def get_variable_groups(value: str) -> Sequence[str]:
598 """Returns a list of variable groups such as $HTTP_SERVERS in a variable."""
599 return __get_variable_groups(value)
600
601
602@lru_cache(maxsize=LRU_CACHE_SIZE)
603def __get_variable_groups(value: str) -> Sequence[str]:
604 entries = get_rule_group_entries(value)
605 variable_groups = []
606 for entry in entries:
607 match = _VARIABLE_GROUP_REGEX.match(entry)
608 if match is not None:
609 variable_groups.append(match.group(1))
610
611 return variable_groups
612
613
[docs]
614def get_rule_body(rule: Rule) -> str:
615 """Returns the body of a rule."""
616 return __get_rule_body(rule)
617
618
619@lru_cache(maxsize=LRU_CACHE_SIZE)
620def __get_rule_body(rule: Rule) -> str:
621 match = _BODY_REGEX.search(rule.raw)
622
623 if match is None:
624 msg = f"Could not extract rule body from rule: {rule.raw}"
625 _logger.critical(msg)
626 raise RuntimeError(msg)
627
628 return match.group(0)
629
630
[docs]
631def is_valid_rule(rule: Rule) -> bool:
632 """Checks if a rule is valid."""
633 if _RULE_REGEX.match(rule.raw) is None:
634 return False
635
636 return True