From 1895a112b0e8070544e3c48b3a56f6b7b07e291e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Tue, 7 Jan 2025 12:55:49 +0300 Subject: [PATCH 01/89] add .idea to gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 09875ae..11ba77a 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ -- 2.54.0 From e901b4e5288de58eaa396177a698a463952e8ef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Tue, 7 Jan 2025 13:28:07 +0300 Subject: [PATCH 02/89] init cidr4_merger --- cidr4_merger.py | 0 tests/test_cidr4_merger.py | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 cidr4_merger.py create mode 100644 tests/test_cidr4_merger.py diff --git a/cidr4_merger.py b/cidr4_merger.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py new file mode 100644 index 0000000..1f3b77f --- /dev/null +++ b/tests/test_cidr4_merger.py @@ -0,0 +1,2 @@ +def test_true(): + assert True -- 2.54.0 From 2a4418fccc28bbdb437d528475876e21d05743c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Tue, 7 Jan 2025 17:35:23 +0300 Subject: [PATCH 03/89] added cidr list merge algorithm --- cidr4.txt | 486 ++++++++++++++++++++++++++++++++++++++++++++++++ cidr4_merger.py | 45 +++++ 2 files changed, 531 insertions(+) create mode 100644 cidr4.txt diff --git a/cidr4.txt b/cidr4.txt new file mode 100644 index 0000000..b3f117f --- /dev/null +++ b/cidr4.txt @@ -0,0 +1,486 @@ +4.78.139.0/24 +23.101.24.0/24 +23.202.231.0/24 +23.217.138.0/24 +23.225.141.0/24 +23.234.30.0/24 +31.13.64.0/24 +31.13.67.0/24 +31.13.68.0/22 +31.13.73.0/24 +31.13.75.0/24 +31.13.76.0/24 +31.13.80.0/21 +31.13.88.0/24 +31.13.90.0/23 +31.13.92.0/24 +31.13.94.0/23 +31.13.96.0/24 +31.13.106.0/24 +31.13.112.0/24 +37.152.2.0/24 +38.121.72.0/24 +39.109.122.0/24 +43.226.16.0/24 +43.245.104.0/24 +45.54.28.0/24 +45.77.186.0/24 +45.114.11.0/24 +45.253.131.0/24 +46.61.154.0/24 +46.134.216.0/24 +47.88.58.0/24 +49.231.55.0/24 +50.23.209.0/24 +50.87.93.0/24 +50.117.117.0/24 +52.58.1.0/24 +52.175.9.0/24 +54.89.135.0/24 +54.144.128.0/24 +54.234.18.0/24 +58.63.233.0/24 +59.18.44.0/24 +59.18.46.0/24 +59.24.3.0/24 +59.188.250.0/24 +61.91.8.0/24 +61.205.119.0/24 +62.0.80.0/24 +64.13.192.0/24 +64.53.242.0/24 +64.233.160.0/21 +64.233.168.0/22 +64.233.176.0/20 +65.49.26.0/24 +65.49.68.0/24 +66.102.1.0/24 +66.220.146.0/23 +66.220.148.0/23 +67.15.100.0/24 +67.15.129.0/24 +67.228.102.0/24 +67.228.235.0/24 +67.230.169.0/24 +69.30.25.0/24 +69.50.221.0/24 +69.63.176.0/24 +69.63.178.0/24 +69.63.180.0/23 +69.63.184.0/24 +69.63.186.0/23 +69.63.190.0/24 +69.162.134.0/24 +69.171.224.0/24 +69.171.227.0/24 +69.171.228.0/23 +69.171.234.0/24 +69.171.242.0/24 +69.171.247.0/24 +69.197.153.0/24 +74.86.3.0/24 +74.86.12.0/24 +74.86.17.0/24 +74.86.118.0/24 +74.86.142.0/24 +74.86.151.0/24 +74.86.226.0/24 +74.86.228.0/24 +74.125.1.0/24 +74.125.2.0/24 +74.125.8.0/24 +74.125.10.0/23 +74.125.13.0/24 +74.125.20.0/23 +74.125.23.0/24 +74.125.24.0/24 +74.125.26.0/24 +74.125.28.0/24 +74.125.31.0/24 +74.125.68.0/22 +74.125.90.0/24 +74.125.96.0/24 +74.125.100.0/24 +74.125.102.0/23 +74.125.104.0/24 +74.125.106.0/24 +74.125.110.0/23 +74.125.124.0/24 +74.125.126.0/23 +74.125.128.0/20 +74.125.153.0/24 +74.125.154.0/23 +74.125.159.0/24 +74.125.164.0/24 +74.125.170.0/24 +74.125.172.0/23 +74.125.192.0/23 +74.125.195.0/24 +74.125.196.0/22 +74.125.200.0/22 +74.125.204.0/23 +74.125.206.0/24 +75.126.33.0/24 +75.126.115.0/24 +75.126.124.0/24 +75.126.135.0/24 +75.126.150.0/24 +75.126.164.0/24 +77.37.252.0/24 +77.120.15.0/24 +80.87.199.0/24 +81.23.20.0/24 +81.23.23.0/24 +81.192.13.0/24 +81.192.191.0/24 +88.191.249.0/24 +90.201.124.0/24 +92.87.232.0/24 +93.179.102.0/24 +94.24.232.0/24 +94.31.189.0/24 +95.59.170.0/24 +96.44.137.0/24 +98.159.108.0/24 +103.39.76.0/24 +103.42.176.0/24 +103.56.16.0/24 +103.73.161.0/24 +103.97.3.0/24 +103.97.176.0/24 +103.200.30.0/23 +103.214.168.0/24 +103.226.246.0/24 +103.228.130.0/24 +103.230.123.0/24 +103.240.180.0/24 +103.240.182.0/24 +103.246.246.0/24 +103.252.114.0/23 +104.16.251.0/24 +104.16.252.0/24 +104.23.124.0/23 +104.31.142.0/24 +104.244.43.0/24 +104.244.45.0/24 +104.244.46.0/24 +107.181.166.0/24 +108.160.161.0/24 +108.160.162.0/23 +108.160.165.0/24 +108.160.166.0/23 +108.160.169.0/24 +108.160.170.0/24 +108.160.172.0/23 +108.177.8.0/21 +108.177.96.0/23 +108.177.98.0/24 +108.177.103.0/24 +108.177.104.0/24 +108.177.111.0/24 +108.177.112.0/24 +108.177.119.0/24 +108.177.120.0/22 +108.177.125.0/24 +108.177.126.0/23 +109.224.41.0/24 +110.164.8.0/24 +111.243.214.0/24 +113.108.239.0/24 +113.171.242.0/24 +114.4.7.0/24 +114.43.24.0/24 +115.126.100.0/24 +116.89.243.0/24 +118.98.30.0/24 +118.98.36.0/24 +118.98.106.0/24 +118.107.180.0/24 +118.184.26.0/24 +118.184.78.0/24 +118.193.202.0/24 +118.193.240.0/24 +119.28.87.0/24 +120.232.233.0/24 +120.232.234.0/24 +120.233.71.0/24 +120.253.250.0/24 +120.253.253.0/24 +120.253.255.0/24 +121.78.42.0/24 +122.10.85.0/24 +122.154.76.0/24 +122.248.226.0/24 +122.252.245.0/24 +124.11.210.0/24 +128.121.146.0/24 +128.121.243.0/24 +128.242.240.0/24 +128.242.245.0/24 +128.242.250.0/24 +130.211.15.0/24 +142.250.0.0/23 +142.250.4.0/24 +142.250.8.0/22 +142.250.12.0/23 +142.250.27.0/24 +142.250.28.0/24 +142.250.30.0/23 +142.250.64.0/20 +142.250.80.0/23 +142.250.96.0/21 +142.250.105.0/24 +142.250.107.0/24 +142.250.110.0/23 +142.250.112.0/22 +142.250.123.0/24 +142.250.125.0/24 +142.250.126.0/24 +142.250.128.0/24 +142.250.136.0/24 +142.250.138.0/24 +142.250.141.0/24 +142.250.142.0/24 +142.250.145.0/24 +142.250.147.0/24 +142.250.148.0/23 +142.250.150.0/24 +142.250.152.0/23 +142.250.157.0/24 +142.250.158.0/23 +142.250.176.0/20 +142.250.192.0/21 +142.250.200.0/23 +142.250.203.0/24 +142.250.204.0/22 +142.250.217.0/24 +142.250.218.0/23 +142.251.0.0/23 +142.251.2.0/24 +142.251.4.0/23 +142.251.6.0/24 +142.251.8.0/23 +142.251.10.0/24 +142.251.12.0/24 +142.251.15.0/24 +142.251.16.0/24 +142.251.18.0/24 +142.251.31.0/24 +142.251.32.0/22 +142.251.36.0/23 +142.251.39.0/24 +142.251.40.0/22 +142.251.45.0/24 +142.251.46.0/23 +142.251.107.0/24 +142.251.111.0/24 +142.251.112.0/24 +142.251.116.0/23 +142.251.120.0/24 +142.251.128.0/23 +142.251.130.0/24 +142.251.132.0/22 +142.251.140.0/22 +142.251.161.0/24 +142.251.162.0/23 +142.251.164.0/22 +142.251.168.0/24 +142.251.170.0/23 +142.251.172.0/22 +142.251.176.0/22 +142.251.180.0/24 +142.251.182.0/23 +142.251.184.0/22 +142.251.188.0/24 +142.251.208.0/23 +142.251.211.0/24 +142.251.214.0/23 +142.251.216.0/24 +142.251.218.0/24 +142.251.220.0/22 +145.255.14.0/24 +148.163.48.0/24 +150.107.3.0/24 +154.0.29.0/24 +154.83.14.0/23 +154.85.102.0/24 +154.92.16.0/24 +156.233.67.0/24 +157.240.0.0/22 +157.240.6.0/23 +157.240.8.0/22 +157.240.12.0/23 +157.240.15.0/24 +157.240.16.0/23 +157.240.18.0/24 +157.240.20.0/23 +159.65.107.0/24 +159.106.121.0/24 +159.138.20.0/24 +162.125.1.0/24 +162.125.2.0/24 +162.125.6.0/23 +162.125.8.0/24 +162.125.17.0/24 +162.125.18.0/24 +162.125.32.0/24 +162.125.34.0/24 +162.125.80.0/24 +162.125.82.0/23 +162.220.12.0/24 +168.143.162.0/24 +168.143.171.0/24 +172.217.0.0/19 +172.217.129.0/24 +172.217.130.0/24 +172.217.133.0/24 +172.217.135.0/24 +172.217.160.0/20 +172.217.192.0/22 +172.217.197.0/24 +172.217.203.0/24 +172.217.204.0/24 +172.217.212.0/24 +172.217.214.0/23 +172.217.218.0/23 +172.217.222.0/24 +172.253.58.0/24 +172.253.62.0/23 +172.253.112.0/21 +172.253.120.0/24 +172.253.122.0/23 +172.253.124.0/22 +173.194.4.0/23 +173.194.12.0/24 +173.194.22.0/24 +173.194.28.0/23 +173.194.31.0/24 +173.194.49.0/24 +173.194.51.0/24 +173.194.54.0/23 +173.194.59.0/24 +173.194.65.0/24 +173.194.66.0/23 +173.194.68.0/23 +173.194.70.0/24 +173.194.73.0/24 +173.194.74.0/24 +173.194.76.0/22 +173.194.135.0/24 +173.194.150.0/24 +173.194.154.0/24 +173.194.161.0/24 +173.194.162.0/23 +173.194.164.0/24 +173.194.166.0/23 +173.194.174.0/23 +173.194.178.0/24 +173.194.182.0/23 +173.194.184.0/23 +173.194.187.0/24 +173.194.188.0/24 +173.194.190.0/23 +173.194.192.0/19 +173.208.182.0/24 +173.231.12.0/24 +173.234.53.0/24 +173.236.182.0/24 +173.236.212.0/24 +173.244.209.0/24 +173.244.217.0/24 +173.252.88.0/24 +173.252.105.0/24 +173.252.108.0/24 +173.252.248.0/24 +173.255.209.0/24 +173.255.213.0/24 +174.36.196.0/24 +174.36.228.0/24 +174.37.54.0/24 +174.37.154.0/24 +174.37.175.0/24 +174.37.243.0/24 +178.151.230.0/24 +178.176.156.0/24 +179.60.193.0/24 +180.163.150.0/23 +182.50.139.0/24 +182.79.251.0/24 +184.72.1.0/24 +184.173.136.0/24 +185.45.6.0/23 +185.60.216.0/24 +185.60.218.0/23 +185.158.208.0/24 +186.208.210.0/24 +187.7.116.0/24 +190.5.235.0/24 +192.133.77.0/24 +192.178.24.0/23 +192.178.27.0/24 +192.178.48.0/23 +192.178.50.0/24 +192.178.52.0/24 +192.178.54.0/24 +192.178.56.0/23 +192.178.128.0/24 +192.178.130.0/24 +193.109.164.0/24 +194.78.0.0/24 +196.49.8.0/24 +198.27.124.0/24 +198.44.185.0/24 +199.16.156.0/24 +199.16.158.0/24 +199.59.148.0/23 +199.59.150.0/24 +199.96.58.0/23 +199.96.61.0/24 +199.96.62.0/23 +199.193.116.0/24 +201.0.223.0/24 +202.53.137.0/24 +202.160.128.0/23 +202.160.130.0/24 +202.169.173.0/24 +202.182.98.0/24 +203.66.182.0/24 +203.111.254.0/24 +203.113.51.0/24 +203.113.189.0/24 +203.208.39.0/24 +203.208.40.0/23 +203.208.43.0/24 +203.208.49.0/24 +203.208.50.0/24 +203.233.96.0/24 +204.79.197.0/24 +205.186.152.0/24 +208.31.254.0/24 +208.43.170.0/24 +208.43.237.0/24 +208.77.47.0/24 +208.101.21.0/24 +208.101.60.0/24 +209.85.144.0/22 +209.85.165.0/24 +209.85.200.0/22 +209.85.224.0/24 +209.85.226.0/24 +209.85.232.0/22 +209.95.56.0/24 +210.56.51.0/24 +210.139.253.0/24 +210.209.84.0/24 +211.104.160.0/24 +212.113.52.0/24 +213.59.210.0/24 +216.58.192.0/20 +216.58.208.0/21 +216.58.217.0/24 +216.58.220.0/22 +216.239.32.0/24 +216.239.34.0/24 +216.239.36.0/24 +216.239.38.0/24 +220.181.174.0/24 diff --git a/cidr4_merger.py b/cidr4_merger.py index e69de29..70ea88d 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -0,0 +1,45 @@ +from copy import deepcopy +from typing import List + +from netaddr import IPNetwork, cidr_merge + + +def get_ips(input_file) -> List[IPNetwork]: + with open(input_file, "r") as file: + data = file.read().splitlines() + return [IPNetwork(line) for line in data] + + +def reduce_prefixlen(ip: IPNetwork, step=1) -> IPNetwork: + """Reduce the CIDR prefix len by step""" + new_ip = IPNetwork(ip) + new_ip.prefixlen -= step + return IPNetwork(new_ip.cidr) + + +def merge_ips(ips_to_reduce: List[IPNetwork], max_len, step=1) -> List[IPNetwork]: + ips = deepcopy(ips_to_reduce) + reduction_limit_reached = False + while len(ips) > max_len and not reduction_limit_reached: + reduced_ips = map(reduce_prefixlen, ips) + merged_ips = cidr_merge(reduced_ips) + if len(merged_ips) > 1: + ips = merged_ips + else: + reduction_limit_reached = True + print("The reduction limit has been reached") + return ips + + +def main(): + file = "cidr4.txt" + ips = get_ips(file) + print(f"{len(ips)=}") + merged_ips = merge_ips(ips, 10) + print(f"{len(merged_ips)=}") + print(", ".join([str(x) for x in merged_ips])) + + +if __name__ == "__main__": + assert reduce_prefixlen(IPNetwork("4.78.139.0/24")) == IPNetwork("4.78.138.0/23") + main() -- 2.54.0 From f7156fb61d76b496d79a38c933cb7f5aefb3781b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 12:16:13 +0300 Subject: [PATCH 04/89] added convert functions --- cidr4_merger.py | 48 +++++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 70ea88d..cddca53 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,45 +1,35 @@ -from copy import deepcopy -from typing import List - -from netaddr import IPNetwork, cidr_merge +import ipaddress +from ipaddress import IPv4Address +from typing import List, Tuple -def get_ips(input_file) -> List[IPNetwork]: +def get_data(input_file): with open(input_file, "r") as file: data = file.read().splitlines() - return [IPNetwork(line) for line in data] + return data -def reduce_prefixlen(ip: IPNetwork, step=1) -> IPNetwork: - """Reduce the CIDR prefix len by step""" - new_ip = IPNetwork(ip) - new_ip.prefixlen -= step - return IPNetwork(new_ip.cidr) +def cidr4_to_binary(cidr4: str) -> Tuple[str, int]: + ip_str, vlsm = cidr4.strip().split("/") + vlsm = int(vlsm) + ipv4 = IPv4Address(ip_str) + binary_ip = bin(int(ipv4))[2:] + return binary_ip, vlsm -def merge_ips(ips_to_reduce: List[IPNetwork], max_len, step=1) -> List[IPNetwork]: - ips = deepcopy(ips_to_reduce) - reduction_limit_reached = False - while len(ips) > max_len and not reduction_limit_reached: - reduced_ips = map(reduce_prefixlen, ips) - merged_ips = cidr_merge(reduced_ips) - if len(merged_ips) > 1: - ips = merged_ips - else: - reduction_limit_reached = True - print("The reduction limit has been reached") - return ips +def binary_to_cidr4(binary_ip: str, vlsm: int) -> str: + int_ip = int(binary_ip, 2) + ip_str = str(ipaddress.IPv4Address(int_ip)) + return f"{ip_str}/{vlsm}" def main(): file = "cidr4.txt" - ips = get_ips(file) - print(f"{len(ips)=}") - merged_ips = merge_ips(ips, 10) - print(f"{len(merged_ips)=}") - print(", ".join([str(x) for x in merged_ips])) + data = get_data(file) if __name__ == "__main__": - assert reduce_prefixlen(IPNetwork("4.78.139.0/24")) == IPNetwork("4.78.138.0/23") + assert cidr4_to_binary("4.78.139.0/24") == ("100010011101000101100000000", 24) + assert binary_to_cidr4("100010011101000101100000000", 24) == "4.78.139.0/24" + main() -- 2.54.0 From 76f2710f7028868cca44e3a0c145fadd0f7a4252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 12:20:22 +0300 Subject: [PATCH 05/89] fix cidr4_to_binary function --- cidr4_merger.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index cddca53..89f58ba 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -14,6 +14,7 @@ def cidr4_to_binary(cidr4: str) -> Tuple[str, int]: vlsm = int(vlsm) ipv4 = IPv4Address(ip_str) binary_ip = bin(int(ipv4))[2:] + binary_ip = binary_ip.zfill(32) return binary_ip, vlsm @@ -26,10 +27,13 @@ def binary_to_cidr4(binary_ip: str, vlsm: int) -> str: def main(): file = "cidr4.txt" data = get_data(file) + bin_ips = list(map(cidr4_to_binary, data)) + for b in bin_ips[:5]: + print(b) if __name__ == "__main__": - assert cidr4_to_binary("4.78.139.0/24") == ("100010011101000101100000000", 24) - assert binary_to_cidr4("100010011101000101100000000", 24) == "4.78.139.0/24" + assert cidr4_to_binary("4.78.139.0/24") == ("00000100010011101000101100000000", 24) + assert binary_to_cidr4("00000100010011101000101100000000", 24) == "4.78.139.0/24" main() -- 2.54.0 From c9c53941eeb50b9ef39f6e2032cf2774a8585f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 12:38:06 +0300 Subject: [PATCH 06/89] add reduce_bin_ip function --- cidr4_merger.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cidr4_merger.py b/cidr4_merger.py index 89f58ba..0605074 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -24,6 +24,14 @@ def binary_to_cidr4(binary_ip: str, vlsm: int) -> str: return f"{ip_str}/{vlsm}" +def reduce_bin_ip(bin_ip: str, vlsm: int) -> Tuple[str, int]: + if vlsm == 0: + return bin_ip, vlsm + new_vlsm = vlsm - 1 + new_bin_ip = bin_ip[:new_vlsm] + "0" * (32 - new_vlsm) + return new_bin_ip, new_vlsm + + def main(): file = "cidr4.txt" data = get_data(file) @@ -35,5 +43,17 @@ def main(): if __name__ == "__main__": assert cidr4_to_binary("4.78.139.0/24") == ("00000100010011101000101100000000", 24) assert binary_to_cidr4("00000100010011101000101100000000", 24) == "4.78.139.0/24" + assert reduce_bin_ip("00000100010011101000101100000000", 24) == ( + "00000100010011101000101000000000", + 23, + ) + assert reduce_bin_ip("10000000000000000000000000000000", 1) == ( + "00000000000000000000000000000000", + 0, + ) + assert reduce_bin_ip("10000000000000000000000000000000", 0) == ( + "10000000000000000000000000000000", + 0, + ) main() -- 2.54.0 From 8b2727ca383d4ab66328086125f28984ec403c8f Mon Sep 17 00:00:00 2001 From: Fedor Lyanguzov Date: Wed, 8 Jan 2025 13:18:58 +0300 Subject: [PATCH 07/89] Test intersecting cidrs will combine --- cidr4_merger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cidr4_merger.py b/cidr4_merger.py index 70ea88d..91998b4 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -42,4 +42,5 @@ def main(): if __name__ == "__main__": assert reduce_prefixlen(IPNetwork("4.78.139.0/24")) == IPNetwork("4.78.138.0/23") + assert merge_ips(list(map(IPNetwork, ["1.0.0.0/24", "1.0.0.1/24", "1.0.0.2/24"])), 1) == [IPNetwork("1.0.0.0/24")] main() -- 2.54.0 From bb269f380aac590ae82200a9533d390d185ab9e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 19:21:44 +0300 Subject: [PATCH 08/89] add rough_merge_ips function --- cidr4_merger.py | 52 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 0605074..990e333 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,4 +1,4 @@ -import ipaddress +from copy import deepcopy from ipaddress import IPv4Address from typing import List, Tuple @@ -20,40 +20,66 @@ def cidr4_to_binary(cidr4: str) -> Tuple[str, int]: def binary_to_cidr4(binary_ip: str, vlsm: int) -> str: int_ip = int(binary_ip, 2) - ip_str = str(ipaddress.IPv4Address(int_ip)) + ip_str = str(IPv4Address(int_ip)) return f"{ip_str}/{vlsm}" def reduce_bin_ip(bin_ip: str, vlsm: int) -> Tuple[str, int]: if vlsm == 0: - return bin_ip, vlsm + return "0" * 32, vlsm new_vlsm = vlsm - 1 new_bin_ip = bin_ip[:new_vlsm] + "0" * (32 - new_vlsm) + if new_bin_ip == "0" * 32: + return "0" * 32, 0 return new_bin_ip, new_vlsm +def rough_merge_ips( + bin_ips: List[Tuple[str, int]], req_len: int +) -> List[Tuple[str, int]]: + ips = deepcopy(bin_ips) + reduction_limit_reached = False + while len(ips) > req_len and not reduction_limit_reached: + max_vlsm = max(ips, key=lambda x: x[1])[1] + ips_to_reduce = [] + merged_ips = [] + for ip_vlsm in ips: + if ip_vlsm[1] == max_vlsm: + ips_to_reduce.append(ip_vlsm) + else: + merged_ips.append(ip_vlsm) + reduced_ips = map(lambda x: reduce_bin_ip(*x), ips_to_reduce) + reduced_ips = set(reduced_ips) + reduced_ips = filter(lambda x: "1" in x[0], reduced_ips) + merged_ips.extend(list(reduced_ips)) + if len(merged_ips) > req_len: + ips = merged_ips + else: + reduction_limit_reached = True + return ips + + def main(): file = "cidr4.txt" data = get_data(file) bin_ips = list(map(cidr4_to_binary, data)) - for b in bin_ips[:5]: - print(b) + rough_merged_bin_ips = rough_merge_ips(bin_ips, 20) + print(f"{len(rough_merged_bin_ips)=}") + sorted_ips = sorted(rough_merged_bin_ips) + for ip in sorted_ips: + print(ip) if __name__ == "__main__": assert cidr4_to_binary("4.78.139.0/24") == ("00000100010011101000101100000000", 24) assert binary_to_cidr4("00000100010011101000101100000000", 24) == "4.78.139.0/24" + assert reduce_bin_ip("00000100010011101000101100000000", 24) == ( "00000100010011101000101000000000", 23, ) - assert reduce_bin_ip("10000000000000000000000000000000", 1) == ( - "00000000000000000000000000000000", - 0, - ) - assert reduce_bin_ip("10000000000000000000000000000000", 0) == ( - "10000000000000000000000000000000", - 0, - ) + assert reduce_bin_ip("10000000000000000000000000000000", 1) == ("0" * 32, 0) + assert reduce_bin_ip("10000000000000000000000000000000", 0) == ("0" * 32, 0) + assert reduce_bin_ip("0" * 32, 0) == ("0" * 32, 0) main() -- 2.54.0 From ddcb043caa6708bad7295513cc07f90a5675b4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 21:16:00 +0300 Subject: [PATCH 09/89] refactoring: represent binary as a string of vlsm length --- cidr4_merger.py | 99 ++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 990e333..0cc7dc7 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -9,77 +9,90 @@ def get_data(input_file): return data -def cidr4_to_binary(cidr4: str) -> Tuple[str, int]: +def cidr4_to_binary(cidr4: str) -> str: ip_str, vlsm = cidr4.strip().split("/") vlsm = int(vlsm) ipv4 = IPv4Address(ip_str) binary_ip = bin(int(ipv4))[2:] binary_ip = binary_ip.zfill(32) - return binary_ip, vlsm + binary = binary_ip[:vlsm] + return binary -def binary_to_cidr4(binary_ip: str, vlsm: int) -> str: +def binary_to_cidr4(binary: str) -> str: + vlsm = len(binary) + binary_ip = binary + "0" * (32 - vlsm) int_ip = int(binary_ip, 2) - ip_str = str(IPv4Address(int_ip)) - return f"{ip_str}/{vlsm}" + ip = IPv4Address(int_ip) + return f"{ip}/{vlsm}" -def reduce_bin_ip(bin_ip: str, vlsm: int) -> Tuple[str, int]: - if vlsm == 0: - return "0" * 32, vlsm - new_vlsm = vlsm - 1 - new_bin_ip = bin_ip[:new_vlsm] + "0" * (32 - new_vlsm) - if new_bin_ip == "0" * 32: - return "0" * 32, 0 - return new_bin_ip, new_vlsm +def reduce_binary(binary: str) -> str: + if len(binary) == 0: + return "" + new_binary = binary[:-1] + if "1" not in new_binary: + return "" + return new_binary -def rough_merge_ips( - bin_ips: List[Tuple[str, int]], req_len: int -) -> List[Tuple[str, int]]: - ips = deepcopy(bin_ips) +def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: + ips = set(deepcopy(binaries)) + non_reducible_ips = set() reduction_limit_reached = False - while len(ips) > req_len and not reduction_limit_reached: - max_vlsm = max(ips, key=lambda x: x[1])[1] - ips_to_reduce = [] - merged_ips = [] - for ip_vlsm in ips: - if ip_vlsm[1] == max_vlsm: - ips_to_reduce.append(ip_vlsm) + while ( + len(ips) > 0 + and len(ips) + len(non_reducible_ips) > req_len + and not reduction_limit_reached + ): + max_vlsm = len(max(ips, key=lambda x: len(x))) + reduced_ips = set() + merged_ips = set() + for ip in ips: + if len(ip) == max_vlsm: + reduced_ip = reduce_binary(ip) + if len(reduced_ip) > 0: + reduced_ips.add(reduced_ip) + else: + non_reducible_ips.add(ip) else: - merged_ips.append(ip_vlsm) - reduced_ips = map(lambda x: reduce_bin_ip(*x), ips_to_reduce) - reduced_ips = set(reduced_ips) - reduced_ips = filter(lambda x: "1" in x[0], reduced_ips) - merged_ips.extend(list(reduced_ips)) - if len(merged_ips) > req_len: + merged_ips.add(ip) + merged_ips.update(reduced_ips) + if len(merged_ips) + len(non_reducible_ips) > req_len: ips = merged_ips else: reduction_limit_reached = True - return ips + + ips.update(non_reducible_ips) + return sorted(ips) def main(): file = "cidr4.txt" + required_len = 20 + data = get_data(file) bin_ips = list(map(cidr4_to_binary, data)) - rough_merged_bin_ips = rough_merge_ips(bin_ips, 20) + rough_merged_bin_ips = rough_merge_binaries(bin_ips, required_len) print(f"{len(rough_merged_bin_ips)=}") - sorted_ips = sorted(rough_merged_bin_ips) - for ip in sorted_ips: + for ip in rough_merged_bin_ips: print(ip) if __name__ == "__main__": - assert cidr4_to_binary("4.78.139.0/24") == ("00000100010011101000101100000000", 24) - assert binary_to_cidr4("00000100010011101000101100000000", 24) == "4.78.139.0/24" + assert cidr4_to_binary("4.78.139.0/24") == "000001000100111010001011" + assert cidr4_to_binary("128.0.0.0/1") == "1" + assert cidr4_to_binary("0.0.0.0/0") == "" - assert reduce_bin_ip("00000100010011101000101100000000", 24) == ( - "00000100010011101000101000000000", - 23, - ) - assert reduce_bin_ip("10000000000000000000000000000000", 1) == ("0" * 32, 0) - assert reduce_bin_ip("10000000000000000000000000000000", 0) == ("0" * 32, 0) - assert reduce_bin_ip("0" * 32, 0) == ("0" * 32, 0) + assert binary_to_cidr4("000001000100111010001011") == "4.78.139.0/24" + assert binary_to_cidr4("1") == "128.0.0.0/1" + assert binary_to_cidr4("") == "0.0.0.0/0" + + assert reduce_binary("000001000100111010001011") == "00000100010011101000101" + assert reduce_binary("1") == "" + assert reduce_binary("0") == "" + assert reduce_binary("") == "" + assert reduce_binary("0001") == "" + assert reduce_binary("0000") == "" main() -- 2.54.0 From 604861aa808d0ca1642332bbf72f3ec82a33b1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 22:29:17 +0300 Subject: [PATCH 10/89] refactoring: represent binary ip as a string of length 32 --- cidr4_merger.py | 59 ++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 0cc7dc7..92e0c63 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,6 +1,8 @@ from copy import deepcopy from ipaddress import IPv4Address from typing import List, Tuple +from itertools import groupby +from collections import defaultdict def get_data(input_file): @@ -15,43 +17,45 @@ def cidr4_to_binary(cidr4: str) -> str: ipv4 = IPv4Address(ip_str) binary_ip = bin(int(ipv4))[2:] binary_ip = binary_ip.zfill(32) - binary = binary_ip[:vlsm] + binary = binary_ip[:vlsm] + "0" * (32 - vlsm) return binary def binary_to_cidr4(binary: str) -> str: - vlsm = len(binary) - binary_ip = binary + "0" * (32 - vlsm) - int_ip = int(binary_ip, 2) + vlsm = binary.rfind("1") + 1 if "1" in binary else 0 + int_ip = int(binary, 2) ip = IPv4Address(int_ip) return f"{ip}/{vlsm}" def reduce_binary(binary: str) -> str: - if len(binary) == 0: - return "" - new_binary = binary[:-1] - if "1" not in new_binary: - return "" - return new_binary + assert len(binary) == 32 + vlsm = binary.rfind("1") + if vlsm == -1: + return binary + return binary[:vlsm] + "0" + binary[vlsm + 1 :] def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ips = set(deepcopy(binaries)) non_reducible_ips = set() reduction_limit_reached = False + max_vlsm = float("inf") while ( len(ips) > 0 and len(ips) + len(non_reducible_ips) > req_len and not reduction_limit_reached + and max_vlsm != -1 ): - max_vlsm = len(max(ips, key=lambda x: len(x))) + ip_with_max_vlsm = max(ips, key=lambda x: x.rfind("1")) + max_vlsm = ip_with_max_vlsm.rfind("1") + print(f"{max_vlsm=}") reduced_ips = set() merged_ips = set() for ip in ips: - if len(ip) == max_vlsm: + if ip.rfind("1") == max_vlsm: reduced_ip = reduce_binary(ip) - if len(reduced_ip) > 0: + if "1" in reduced_ip: reduced_ips.add(reduced_ip) else: non_reducible_ips.add(ip) @@ -62,6 +66,7 @@ def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ips = merged_ips else: reduction_limit_reached = True + print(f"{len(ips)=}") ips.update(non_reducible_ips) return sorted(ips) @@ -73,6 +78,7 @@ def main(): data = get_data(file) bin_ips = list(map(cidr4_to_binary, data)) + rough_merged_bin_ips = rough_merge_binaries(bin_ips, required_len) print(f"{len(rough_merged_bin_ips)=}") for ip in rough_merged_bin_ips: @@ -80,19 +86,22 @@ def main(): if __name__ == "__main__": - assert cidr4_to_binary("4.78.139.0/24") == "000001000100111010001011" - assert cidr4_to_binary("128.0.0.0/1") == "1" - assert cidr4_to_binary("0.0.0.0/0") == "" + assert cidr4_to_binary("4.78.139.0/24") == "00000100010011101000101100000000" + assert cidr4_to_binary("128.0.0.0/1") == "10000000000000000000000000000000" + assert cidr4_to_binary("0.0.0.0/0") == "0" * 32 + assert cidr4_to_binary("0.8.0.0/13") == "00000000000010000000000000000000" - assert binary_to_cidr4("000001000100111010001011") == "4.78.139.0/24" - assert binary_to_cidr4("1") == "128.0.0.0/1" - assert binary_to_cidr4("") == "0.0.0.0/0" + assert binary_to_cidr4("00000100010011101000101100000000") == "4.78.139.0/24" + assert binary_to_cidr4("10000000000000000000000000000000") == "128.0.0.0/1" + assert binary_to_cidr4("00000000000000000000000000000000") == "0.0.0.0/0" + assert binary_to_cidr4("00000000000010000000000000000000") == "0.8.0.0/13" - assert reduce_binary("000001000100111010001011") == "00000100010011101000101" - assert reduce_binary("1") == "" - assert reduce_binary("0") == "" - assert reduce_binary("") == "" - assert reduce_binary("0001") == "" - assert reduce_binary("0000") == "" + assert ( + reduce_binary("00000100010011101000101100000000") + == "00000100010011101000101000000000" + ) + assert reduce_binary("10000000000000000000000000000000") == "0" * 32 + assert reduce_binary("0" * 32) == "0" * 32 + assert reduce_binary("00000000000010000000000000000000") == "0" * 32 main() -- 2.54.0 From fab8786ff843cf25595163eccc7ced27e749ddc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 22:30:13 +0300 Subject: [PATCH 11/89] cleanup --- cidr4_merger.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 92e0c63..40be3d0 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -49,7 +49,6 @@ def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ): ip_with_max_vlsm = max(ips, key=lambda x: x.rfind("1")) max_vlsm = ip_with_max_vlsm.rfind("1") - print(f"{max_vlsm=}") reduced_ips = set() merged_ips = set() for ip in ips: @@ -66,7 +65,6 @@ def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ips = merged_ips else: reduction_limit_reached = True - print(f"{len(ips)=}") ips.update(non_reducible_ips) return sorted(ips) -- 2.54.0 From d514d4f5e060ab1cf8548fc91030d03d4f0c6bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Wed, 8 Jan 2025 23:13:57 +0300 Subject: [PATCH 12/89] added smooth_merge_binaries function --- cidr4_merger.py | 52 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/cidr4_merger.py b/cidr4_merger.py index 40be3d0..fbbebed 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -3,6 +3,7 @@ from ipaddress import IPv4Address from typing import List, Tuple from itertools import groupby from collections import defaultdict +import cProfile def get_data(input_file): @@ -70,6 +71,49 @@ def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: return sorted(ips) +def smooth_merge_binaries(binaries: List[str], req_len: int) -> List[str]: + non_reducible_ips = set() + ips = set() + for ip in binaries: + if "1" not in reduce_binary(ip): + non_reducible_ips.add(ip) + else: + ips.add(ip) + + while len(ips) > 0 and len(ips) + len(non_reducible_ips) > req_len: + print(f"{req_len=} {len(ips) + len(non_reducible_ips)=}") + + group_dict = defaultdict(list) + for ip in ips: + group_dict[ip.rfind("1")].append(ip) + + max_len = max(group_dict) + lst = group_dict[max_len] + for x, y in zip(lst, lst[1:]): + i = x.rfind("1") + if y.startswith(x[:i]): + ips.remove(x) + ips.remove(y) + ips.add(reduce_binary(x)) + break + else: + x = lst[0] + ips.remove(x) + ips.add(reduce_binary(x)) + + _ips = ips | non_reducible_ips + non_reducible_ips = set() + ips = set() + for ip in _ips: + if "1" not in reduce_binary(ip): + non_reducible_ips.add(ip) + else: + ips.add(ip) + + ips.update(non_reducible_ips) + return sorted(ips) + + def main(): file = "cidr4.txt" required_len = 20 @@ -82,6 +126,13 @@ def main(): for ip in rough_merged_bin_ips: print(ip) + print("#" * 128) + + smooth_merged_bin_ips = smooth_merge_binaries(rough_merged_bin_ips, required_len) + print(f"{len(smooth_merged_bin_ips)=}") + for ip in smooth_merged_bin_ips: + print(ip) + if __name__ == "__main__": assert cidr4_to_binary("4.78.139.0/24") == "00000100010011101000101100000000" @@ -103,3 +154,4 @@ if __name__ == "__main__": assert reduce_binary("00000000000010000000000000000000") == "0" * 32 main() + # cProfile.run("main()") -- 2.54.0 From 7d89eee92f18d21c728a7b2636d25eb7b437c32e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Thu, 9 Jan 2025 01:09:29 +0300 Subject: [PATCH 13/89] =?UTF-8?q?fix=20rough=5Fmerge=5Fbinaries=20fun?= =?UTF-8?q?=D1=81,=20previously=20the=20function=20returned=20networks=20a?= =?UTF-8?q?nd=20subnets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cidr4_merger.py | 57 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index fbbebed..0626792 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -37,37 +37,41 @@ def reduce_binary(binary: str) -> str: return binary[:vlsm] + "0" + binary[vlsm + 1 :] +def get_mask(binary: str) -> str: + i = binary.rfind("1") + return binary[: i + 1] + + def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ips = set(deepcopy(binaries)) - non_reducible_ips = set() reduction_limit_reached = False max_vlsm = float("inf") - while ( - len(ips) > 0 - and len(ips) + len(non_reducible_ips) > req_len - and not reduction_limit_reached - and max_vlsm != -1 - ): + while len(ips) > req_len and not reduction_limit_reached and max_vlsm != -1: ip_with_max_vlsm = max(ips, key=lambda x: x.rfind("1")) max_vlsm = ip_with_max_vlsm.rfind("1") reduced_ips = set() merged_ips = set() for ip in ips: if ip.rfind("1") == max_vlsm: - reduced_ip = reduce_binary(ip) - if "1" in reduced_ip: - reduced_ips.add(reduced_ip) - else: - non_reducible_ips.add(ip) + reduced_ips.add(reduce_binary(ip)) else: merged_ips.add(ip) + + filtered_set = set() + for r_ip in reduced_ips: + mask = get_mask(r_ip) + for ip in merged_ips: + if ip.startswith(mask): + filtered_set.add(r_ip) + + reduced_ips -= filtered_set merged_ips.update(reduced_ips) - if len(merged_ips) + len(non_reducible_ips) > req_len: + + if len(merged_ips) > req_len: ips = merged_ips else: reduction_limit_reached = True - ips.update(non_reducible_ips) return sorted(ips) @@ -81,7 +85,7 @@ def smooth_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ips.add(ip) while len(ips) > 0 and len(ips) + len(non_reducible_ips) > req_len: - print(f"{req_len=} {len(ips) + len(non_reducible_ips)=}") + # print(f"{req_len=} {len(ips) + len(non_reducible_ips)=}") group_dict = defaultdict(list) for ip in ips: @@ -120,18 +124,26 @@ def main(): data = get_data(file) bin_ips = list(map(cidr4_to_binary, data)) + # for ip in bin_ips: + # print(ip) + # print("#" * 128) rough_merged_bin_ips = rough_merge_binaries(bin_ips, required_len) print(f"{len(rough_merged_bin_ips)=}") for ip in rough_merged_bin_ips: print(ip) - print("#" * 128) + # print("#" * 128) + # + # smooth_merged_bin_ips = smooth_merge_binaries(rough_merged_bin_ips, required_len) + # print(f"{len(smooth_merged_bin_ips)=}") + # for ip in smooth_merged_bin_ips: + # print(ip) - smooth_merged_bin_ips = smooth_merge_binaries(rough_merged_bin_ips, required_len) - print(f"{len(smooth_merged_bin_ips)=}") - for ip in smooth_merged_bin_ips: - print(ip) + # smooth_merged_bin_ips = smooth_merge_binaries(bin_ips, required_len) + # print(f"{len(smooth_merged_bin_ips)=}") + # for ip in smooth_merged_bin_ips: + # print(ip) if __name__ == "__main__": @@ -153,5 +165,10 @@ if __name__ == "__main__": assert reduce_binary("0" * 32) == "0" * 32 assert reduce_binary("00000000000010000000000000000000") == "0" * 32 + assert get_mask("00000100010011101000101100000000") == "000001000100111010001011" + assert get_mask("10000000000000000000000000000000") == "1" + assert get_mask("00000000000000000000000000000000") == "" + assert get_mask("00000000000010000000000000000000") == "0000000000001" + main() # cProfile.run("main()") -- 2.54.0 From 722bd5a751a5b914087147fe5a06f2375b914dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Thu, 9 Jan 2025 13:00:37 +0300 Subject: [PATCH 14/89] add remove_ips_with_subnets function and really fix merge functions --- cidr4_merger.py | 94 +++++++++++++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 43 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 0626792..1036198 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,6 +1,6 @@ from copy import deepcopy from ipaddress import IPv4Address -from typing import List, Tuple +from typing import List, Tuple, Set from itertools import groupby from collections import defaultdict import cProfile @@ -42,6 +42,19 @@ def get_mask(binary: str) -> str: return binary[: i + 1] +def remove_ips_with_subnets(binaries: Set[str]) -> Set[str]: + """Убрать ip, у которых есть подсети""" + sorted_binaries = sorted(binaries) + i = 0 + while i < len(sorted_binaries) - 1: + mask = get_mask(sorted_binaries[i]) + if sorted_binaries[i + 1].startswith(mask): + del sorted_binaries[i] + else: + i += 1 + return set(sorted_binaries) + + def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ips = set(deepcopy(binaries)) reduction_limit_reached = False @@ -50,23 +63,14 @@ def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ip_with_max_vlsm = max(ips, key=lambda x: x.rfind("1")) max_vlsm = ip_with_max_vlsm.rfind("1") reduced_ips = set() - merged_ips = set() + other_ips = set() for ip in ips: if ip.rfind("1") == max_vlsm: reduced_ips.add(reduce_binary(ip)) else: - merged_ips.add(ip) - - filtered_set = set() - for r_ip in reduced_ips: - mask = get_mask(r_ip) - for ip in merged_ips: - if ip.startswith(mask): - filtered_set.add(r_ip) - - reduced_ips -= filtered_set - merged_ips.update(reduced_ips) + other_ips.add(ip) + merged_ips = remove_ips_with_subnets(reduced_ips | other_ips) if len(merged_ips) > req_len: ips = merged_ips else: @@ -76,45 +80,29 @@ def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: def smooth_merge_binaries(binaries: List[str], req_len: int) -> List[str]: - non_reducible_ips = set() - ips = set() - for ip in binaries: - if "1" not in reduce_binary(ip): - non_reducible_ips.add(ip) - else: - ips.add(ip) - - while len(ips) > 0 and len(ips) + len(non_reducible_ips) > req_len: - # print(f"{req_len=} {len(ips) + len(non_reducible_ips)=}") + ips = set(deepcopy(binaries)) + while len(ips) > req_len: group_dict = defaultdict(list) for ip in ips: group_dict[ip.rfind("1")].append(ip) max_len = max(group_dict) - lst = group_dict[max_len] - for x, y in zip(lst, lst[1:]): - i = x.rfind("1") - if y.startswith(x[:i]): + sorted_group = sorted(group_dict[max_len]) + for x, y in zip(sorted_group, sorted_group[1:]): + mask = get_mask(x) + if y.startswith(mask): ips.remove(x) ips.remove(y) ips.add(reduce_binary(x)) break else: - x = lst[0] + x = sorted_group[0] ips.remove(x) ips.add(reduce_binary(x)) - _ips = ips | non_reducible_ips - non_reducible_ips = set() - ips = set() - for ip in _ips: - if "1" not in reduce_binary(ip): - non_reducible_ips.add(ip) - else: - ips.add(ip) + ips = remove_ips_with_subnets(ips) - ips.update(non_reducible_ips) return sorted(ips) @@ -133,12 +121,12 @@ def main(): for ip in rough_merged_bin_ips: print(ip) - # print("#" * 128) - # - # smooth_merged_bin_ips = smooth_merge_binaries(rough_merged_bin_ips, required_len) - # print(f"{len(smooth_merged_bin_ips)=}") - # for ip in smooth_merged_bin_ips: - # print(ip) + print("\n" + "#" * 128 + "\n") + + smooth_merged_bin_ips = smooth_merge_binaries(rough_merged_bin_ips, required_len) + print(f"{len(smooth_merged_bin_ips)=}") + for ip in smooth_merged_bin_ips: + print(ip) # smooth_merged_bin_ips = smooth_merge_binaries(bin_ips, required_len) # print(f"{len(smooth_merged_bin_ips)=}") @@ -170,5 +158,25 @@ if __name__ == "__main__": assert get_mask("00000000000000000000000000000000") == "" assert get_mask("00000000000010000000000000000000") == "0000000000001" + assert remove_ips_with_subnets( + { + "0001000", + "0001100", + "0001110", + "0001111", + "0101000", + } + ) == { + "0001111", + "0101000", + } + + assert remove_ips_with_subnets({"0001000"}) == {"0001000"} + assert remove_ips_with_subnets({"1111000", "0101000"}) == { + "1111000", + "0101000", + } + assert remove_ips_with_subnets(set()) == set() + main() # cProfile.run("main()") -- 2.54.0 From 99da7ea7262bfe391663eb427e44de1dbedf2f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Thu, 9 Jan 2025 13:46:59 +0300 Subject: [PATCH 15/89] cleanup --- cidr4_merger.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 1036198..62fb9fa 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,9 +1,8 @@ +import cProfile +from collections import defaultdict from copy import deepcopy from ipaddress import IPv4Address -from typing import List, Tuple, Set -from itertools import groupby -from collections import defaultdict -import cProfile +from typing import List, Set def get_data(input_file): @@ -18,8 +17,7 @@ def cidr4_to_binary(cidr4: str) -> str: ipv4 = IPv4Address(ip_str) binary_ip = bin(int(ipv4))[2:] binary_ip = binary_ip.zfill(32) - binary = binary_ip[:vlsm] + "0" * (32 - vlsm) - return binary + return binary_ip[:vlsm] + "0" * (32 - vlsm) def binary_to_cidr4(binary: str) -> str: @@ -30,7 +28,6 @@ def binary_to_cidr4(binary: str) -> str: def reduce_binary(binary: str) -> str: - assert len(binary) == 32 vlsm = binary.rfind("1") if vlsm == -1: return binary @@ -43,7 +40,6 @@ def get_mask(binary: str) -> str: def remove_ips_with_subnets(binaries: Set[str]) -> Set[str]: - """Убрать ip, у которых есть подсети""" sorted_binaries = sorted(binaries) i = 0 while i < len(sorted_binaries) - 1: @@ -58,7 +54,7 @@ def remove_ips_with_subnets(binaries: Set[str]) -> Set[str]: def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: ips = set(deepcopy(binaries)) reduction_limit_reached = False - max_vlsm = float("inf") + max_vlsm = None while len(ips) > req_len and not reduction_limit_reached and max_vlsm != -1: ip_with_max_vlsm = max(ips, key=lambda x: x.rfind("1")) max_vlsm = ip_with_max_vlsm.rfind("1") @@ -178,5 +174,5 @@ if __name__ == "__main__": } assert remove_ips_with_subnets(set()) == set() - main() - # cProfile.run("main()") + # main() + cProfile.run("main()") -- 2.54.0 From 113b6511672492a2dc2ce747daec235e16ce93b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Thu, 9 Jan 2025 14:19:00 +0300 Subject: [PATCH 16/89] speed up remove_ips_with_subnets function --- cidr4_merger.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 62fb9fa..ef7b1b8 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -41,14 +41,13 @@ def get_mask(binary: str) -> str: def remove_ips_with_subnets(binaries: Set[str]) -> Set[str]: sorted_binaries = sorted(binaries) - i = 0 - while i < len(sorted_binaries) - 1: - mask = get_mask(sorted_binaries[i]) - if sorted_binaries[i + 1].startswith(mask): - del sorted_binaries[i] - else: - i += 1 - return set(sorted_binaries) + result = set() + for x, y in zip(sorted_binaries, sorted_binaries[1:]): + mask = get_mask(x) + if not y.startswith(mask): + result.add(x) + result.add(sorted_binaries[-1]) + return result def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: @@ -104,7 +103,7 @@ def smooth_merge_binaries(binaries: List[str], req_len: int) -> List[str]: def main(): file = "cidr4.txt" - required_len = 20 + required_len = 15 data = get_data(file) bin_ips = list(map(cidr4_to_binary, data)) @@ -172,7 +171,6 @@ if __name__ == "__main__": "1111000", "0101000", } - assert remove_ips_with_subnets(set()) == set() # main() cProfile.run("main()") -- 2.54.0 From 9148b9be327fc112441ff2d93f385891c11e31df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 07:00:22 +0300 Subject: [PATCH 17/89] init new algorithm --- cidr4_merger.py | 157 +----------------------------------------------- 1 file changed, 2 insertions(+), 155 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index ef7b1b8..0b767fb 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -11,166 +11,13 @@ def get_data(input_file): return data -def cidr4_to_binary(cidr4: str) -> str: - ip_str, vlsm = cidr4.strip().split("/") - vlsm = int(vlsm) - ipv4 = IPv4Address(ip_str) - binary_ip = bin(int(ipv4))[2:] - binary_ip = binary_ip.zfill(32) - return binary_ip[:vlsm] + "0" * (32 - vlsm) - - -def binary_to_cidr4(binary: str) -> str: - vlsm = binary.rfind("1") + 1 if "1" in binary else 0 - int_ip = int(binary, 2) - ip = IPv4Address(int_ip) - return f"{ip}/{vlsm}" - - -def reduce_binary(binary: str) -> str: - vlsm = binary.rfind("1") - if vlsm == -1: - return binary - return binary[:vlsm] + "0" + binary[vlsm + 1 :] - - -def get_mask(binary: str) -> str: - i = binary.rfind("1") - return binary[: i + 1] - - -def remove_ips_with_subnets(binaries: Set[str]) -> Set[str]: - sorted_binaries = sorted(binaries) - result = set() - for x, y in zip(sorted_binaries, sorted_binaries[1:]): - mask = get_mask(x) - if not y.startswith(mask): - result.add(x) - result.add(sorted_binaries[-1]) - return result - - -def rough_merge_binaries(binaries: List[str], req_len: int) -> List[str]: - ips = set(deepcopy(binaries)) - reduction_limit_reached = False - max_vlsm = None - while len(ips) > req_len and not reduction_limit_reached and max_vlsm != -1: - ip_with_max_vlsm = max(ips, key=lambda x: x.rfind("1")) - max_vlsm = ip_with_max_vlsm.rfind("1") - reduced_ips = set() - other_ips = set() - for ip in ips: - if ip.rfind("1") == max_vlsm: - reduced_ips.add(reduce_binary(ip)) - else: - other_ips.add(ip) - - merged_ips = remove_ips_with_subnets(reduced_ips | other_ips) - if len(merged_ips) > req_len: - ips = merged_ips - else: - reduction_limit_reached = True - - return sorted(ips) - - -def smooth_merge_binaries(binaries: List[str], req_len: int) -> List[str]: - ips = set(deepcopy(binaries)) - - while len(ips) > req_len: - group_dict = defaultdict(list) - for ip in ips: - group_dict[ip.rfind("1")].append(ip) - - max_len = max(group_dict) - sorted_group = sorted(group_dict[max_len]) - for x, y in zip(sorted_group, sorted_group[1:]): - mask = get_mask(x) - if y.startswith(mask): - ips.remove(x) - ips.remove(y) - ips.add(reduce_binary(x)) - break - else: - x = sorted_group[0] - ips.remove(x) - ips.add(reduce_binary(x)) - - ips = remove_ips_with_subnets(ips) - - return sorted(ips) - - def main(): file = "cidr4.txt" required_len = 15 data = get_data(file) - bin_ips = list(map(cidr4_to_binary, data)) - # for ip in bin_ips: - # print(ip) - # print("#" * 128) - - rough_merged_bin_ips = rough_merge_binaries(bin_ips, required_len) - print(f"{len(rough_merged_bin_ips)=}") - for ip in rough_merged_bin_ips: - print(ip) - - print("\n" + "#" * 128 + "\n") - - smooth_merged_bin_ips = smooth_merge_binaries(rough_merged_bin_ips, required_len) - print(f"{len(smooth_merged_bin_ips)=}") - for ip in smooth_merged_bin_ips: - print(ip) - - # smooth_merged_bin_ips = smooth_merge_binaries(bin_ips, required_len) - # print(f"{len(smooth_merged_bin_ips)=}") - # for ip in smooth_merged_bin_ips: - # print(ip) if __name__ == "__main__": - assert cidr4_to_binary("4.78.139.0/24") == "00000100010011101000101100000000" - assert cidr4_to_binary("128.0.0.0/1") == "10000000000000000000000000000000" - assert cidr4_to_binary("0.0.0.0/0") == "0" * 32 - assert cidr4_to_binary("0.8.0.0/13") == "00000000000010000000000000000000" - - assert binary_to_cidr4("00000100010011101000101100000000") == "4.78.139.0/24" - assert binary_to_cidr4("10000000000000000000000000000000") == "128.0.0.0/1" - assert binary_to_cidr4("00000000000000000000000000000000") == "0.0.0.0/0" - assert binary_to_cidr4("00000000000010000000000000000000") == "0.8.0.0/13" - - assert ( - reduce_binary("00000100010011101000101100000000") - == "00000100010011101000101000000000" - ) - assert reduce_binary("10000000000000000000000000000000") == "0" * 32 - assert reduce_binary("0" * 32) == "0" * 32 - assert reduce_binary("00000000000010000000000000000000") == "0" * 32 - - assert get_mask("00000100010011101000101100000000") == "000001000100111010001011" - assert get_mask("10000000000000000000000000000000") == "1" - assert get_mask("00000000000000000000000000000000") == "" - assert get_mask("00000000000010000000000000000000") == "0000000000001" - - assert remove_ips_with_subnets( - { - "0001000", - "0001100", - "0001110", - "0001111", - "0101000", - } - ) == { - "0001111", - "0101000", - } - - assert remove_ips_with_subnets({"0001000"}) == {"0001000"} - assert remove_ips_with_subnets({"1111000", "0101000"}) == { - "1111000", - "0101000", - } - - # main() - cProfile.run("main()") + main() + # cProfile.run("main()") -- 2.54.0 From dfd02e3da552874c41eb320c1954efae8f21e720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 07:24:48 +0300 Subject: [PATCH 18/89] add convert from cidr4 to node --- cidr4_merger.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 0b767fb..e39134e 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -2,7 +2,7 @@ import cProfile from collections import defaultdict from copy import deepcopy from ipaddress import IPv4Address -from typing import List, Set +from typing import List, Set, Tuple def get_data(input_file): @@ -11,13 +11,28 @@ def get_data(input_file): return data +def cidr4_to_node(cidr4: str) -> Tuple[int, int, int]: + ip, mask_len = cidr4.strip().split("/") + mask_len = int(mask_len) + added_ips_number = 0 + a, b, c, d = ip.split(".") + a, b, c, d = int(a), int(b), int(c), int(d) + ip_value = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 + return ip_value, mask_len, added_ips_number + + def main(): file = "cidr4.txt" required_len = 15 data = get_data(file) + nodes = sorted(map(cidr4_to_node, data)) + for n in nodes: + print(n) if __name__ == "__main__": + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) + main() # cProfile.run("main()") -- 2.54.0 From 9377d2313430252ac574e1df7e0727e5e99d64a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 08:08:08 +0300 Subject: [PATCH 19/89] add convert data to nodes --- cidr4_merger.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index e39134e..dfe9a62 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,8 +1,6 @@ import cProfile -from collections import defaultdict -from copy import deepcopy -from ipaddress import IPv4Address -from typing import List, Set, Tuple + +Node = tuple[int, int, int] def get_data(input_file): @@ -11,28 +9,32 @@ def get_data(input_file): return data -def cidr4_to_node(cidr4: str) -> Tuple[int, int, int]: +def cidr4_to_node(cidr4: str) -> Node: ip, mask_len = cidr4.strip().split("/") mask_len = int(mask_len) added_ips_number = 0 - a, b, c, d = ip.split(".") - a, b, c, d = int(a), int(b), int(c), int(d) + a, b, c, d = list(map(int, ip.split("."))) ip_value = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 return ip_value, mask_len, added_ips_number +def data_to_nodes(data: list[str]) -> list[Node]: + return sorted(map(cidr4_to_node, data)) + + def main(): file = "cidr4.txt" required_len = 15 data = get_data(file) - nodes = sorted(map(cidr4_to_node, data)) + nodes = data_to_nodes(data) for n in nodes: print(n) if __name__ == "__main__": assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) main() # cProfile.run("main()") -- 2.54.0 From 3a5a195d3da1370a20fd4a602e93c77e6bbe0ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 09:45:35 +0300 Subject: [PATCH 20/89] add have_same_parent func --- cidr4_merger.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index dfe9a62..ca7eabc 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,4 +1,5 @@ import cProfile +from typing import Optional Node = tuple[int, int, int] @@ -22,6 +23,27 @@ def data_to_nodes(data: list[str]) -> list[Node]: return sorted(map(cidr4_to_node, data)) +def get_mask(node: Node) -> int: + value, mask_len, _ = node + x = (2**mask_len - 1) << (32 - mask_len) + mask = value & x + return mask + + +def get_parent_mask(node: Node) -> Optional[int]: + if node[1] == 0: + return None + return get_mask((node[0], node[1] - 1, node[2])) + + +def have_same_parent(a: Node, b: Node) -> bool: + return a[1] == b[1] and get_parent_mask(a) == get_parent_mask(b) + + +def merge_nodes(nodes: list[Node]) -> list[Node]: + pass + + def main(): file = "cidr4.txt" required_len = 15 @@ -36,5 +58,37 @@ if __name__ == "__main__": assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) - main() + bin_a = "10011000000000001000010000010000" + assert len(bin_a) == 32 + value_a = int(bin_a, 2) + + bin_b = "10011100000000000000000000101011" + assert len(bin_b) == 32 + value_b = int(bin_b, 2) + + bin_c = "10011000000000000000000000000000" + assert len(bin_c) == 32 + value_c = int(bin_c, 2) + + bin_d = "11111100000000000000000000000000" + assert len(bin_c) == 32 + value_d = int(bin_d, 2) + + assert get_mask((value_a, 5, 0)) == value_c + assert get_mask((value_b, 5, 0)) == value_c + assert get_mask((0, 1, 0)) == 0 + assert get_mask((0, 0, 0)) == 0 + + assert get_parent_mask((value_a, 6, 0)) == value_c + assert get_parent_mask((value_b, 6, 0)) == value_c + assert get_parent_mask((0, 1, 0)) == 0 + assert get_parent_mask((0, 0, 0)) is None + + assert have_same_parent((value_a, 6, 0), (value_b, 6, 0)) is True + assert have_same_parent((value_a, 6, 0), (value_b, 5, 0)) is False + assert have_same_parent((value_a, 6, 0), (value_d, 6, 0)) is False + assert have_same_parent((value_a, 6, 0), (0, 1, 0)) is False + assert have_same_parent((value_a, 6, 0), (0, 0, 0)) is False + + # main() # cProfile.run("main()") -- 2.54.0 From 020836d33dae06b37d19fbfc5adccc047a02dad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 14:40:21 +0300 Subject: [PATCH 21/89] add sort_nodes func --- cidr4_merger.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index ca7eabc..723ce9d 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -19,8 +19,12 @@ def cidr4_to_node(cidr4: str) -> Node: return ip_value, mask_len, added_ips_number +def sort_nodes(nodes: list[Node]) -> list[Node]: + return sorted(nodes, key=lambda x: (x[1], x[0])) + + def data_to_nodes(data: list[str]) -> list[Node]: - return sorted(map(cidr4_to_node, data)) + return sort_nodes(map(cidr4_to_node, data)) def get_mask(node: Node) -> int: @@ -90,5 +94,19 @@ if __name__ == "__main__": assert have_same_parent((value_a, 6, 0), (0, 1, 0)) is False assert have_same_parent((value_a, 6, 0), (0, 0, 0)) is False - # main() + assert sort_nodes( + [ + (401219072, 24, 0), + (2899902464, 19, 0), + (400657664, 24, 0), + (520969728, 23, 0), + ] + ) == [ + (2899902464, 19, 0), + (520969728, 23, 0), + (400657664, 24, 0), + (401219072, 24, 0), + ] + + main() # cProfile.run("main()") -- 2.54.0 From 5a4166095b3b36c54434c442be2c195559837231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 22:30:08 +0300 Subject: [PATCH 22/89] add reduce_nodes function --- cidr4_merger.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 125 insertions(+), 4 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 723ce9d..5c3e5e7 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,5 +1,7 @@ import cProfile from typing import Optional +from collections import defaultdict +from itertools import groupby Node = tuple[int, int, int] @@ -44,7 +46,60 @@ def have_same_parent(a: Node, b: Node) -> bool: return a[1] == b[1] and get_parent_mask(a) == get_parent_mask(b) -def merge_nodes(nodes: list[Node]) -> list[Node]: +def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: + max_mask_len = max(nodes, key=lambda x: x[1])[1] + return sort_nodes(filter(lambda x: x[1] == max_mask_len, nodes)) + + +def get_parent(a: Node, b: Node = None) -> Node: + ip_value = get_parent_mask(a) + mask_len = a[1] - 1 + added_ips_number = a[2] + 2 ** (32 - a[1]) + if b: + assert have_same_parent(a, b), "ноды должны быть соседями" + added_ips_number = a[2] + b[2] + return ip_value, mask_len, added_ips_number + + +def reduce_nodes(nodes: list[Node]) -> list[Node]: + group = get_group_with_max_mask_len(nodes) + + neighbours = [] + loners = [] + i = 0 + while i < len(group) - 1: + a = group[i] + b = group[i + 1] + if have_same_parent(a, b): + neighbours.append((a, b)) + i += 2 + else: + loners.append((a,)) + i += 1 + # может лучше проверять левый или правый + if i == len(group) - 1: + loners.append(group[i]) + + if neighbours: + zipped = zip(neighbours, map(lambda x: get_parent(x[0], x[1]), neighbours)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + (a, b), parent = min_zipped + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + elif loners: + zipped = zip(loners, map(get_parent, loners)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + a, parent = min_zipped + nodes.remove(a) + nodes.append(parent) + else: + assert False, "Error" + + return sort_nodes(nodes) + + +def merge_nodes(nodes: list[Node], required_len: int) -> list[Node]: pass @@ -54,8 +109,10 @@ def main(): data = get_data(file) nodes = data_to_nodes(data) - for n in nodes: - print(n) + # for n in nodes: + # print(n) + + merged_nodes = merge_nodes(nodes, required_len) if __name__ == "__main__": @@ -108,5 +165,69 @@ if __name__ == "__main__": (401219072, 24, 0), ] - main() + assert get_group_with_max_mask_len( + [ + (2899902464, 19, 0), + (520969728, 23, 0), + (400657664, 24, 0), + (401219072, 24, 0), + ] + ) == [(400657664, 24, 0), (401219072, 24, 0)] + assert get_group_with_max_mask_len( + [ + (401219072, 24, 0), + (2899902464, 19, 0), + (520969728, 23, 0), + ] + ) == [(401219072, 24, 0)] + + assert get_parent((0, 2, 12), (1073741824, 2, 3)) == (0, 1, 15) + assert get_parent((2147483648, 2, 1), (3221225472, 2, 2)) == (2147483648, 1, 3) + + assert reduce_nodes( + [ + (0, 0, 0), + (0, 2, 12), + (1073741824, 2, 3), + ] + ) == [ + (0, 0, 0), + (0, 1, 15), + ] + + assert reduce_nodes( + [ + (0, 0, 0), + (0, 2, 12), + (1073741824, 2, 3), + (2147483648, 2, 1), + (3221225472, 2, 2), + ] + ) == [ + (0, 0, 0), + (2147483648, 1, 3), + (0, 2, 12), + (1073741824, 2, 3), + ] + + assert reduce_nodes( + [ + (0, 2, 12), + (2147483648, 1, 0), + ] + ) == [ + (0, 1, 12 + 2**30), + (2147483648, 1, 0), + ] + + assert reduce_nodes( + [ + (0, 1, 12 + 2**30), + (2147483648, 1, 0), + ] + ) == [ + (0, 0, 12 + 2**30), + ] + + # main() # cProfile.run("main()") -- 2.54.0 From aa75fd7ba68e8ee322d58910928234183e90d207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 22:39:23 +0300 Subject: [PATCH 23/89] cleanup --- cidr4_merger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 5c3e5e7..1bce248 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,7 +1,5 @@ import cProfile from typing import Optional -from collections import defaultdict -from itertools import groupby Node = tuple[int, int, int] @@ -26,7 +24,7 @@ def sort_nodes(nodes: list[Node]) -> list[Node]: def data_to_nodes(data: list[str]) -> list[Node]: - return sort_nodes(map(cidr4_to_node, data)) + return list(map(cidr4_to_node, data)) def get_mask(node: Node) -> int: @@ -76,7 +74,6 @@ def reduce_nodes(nodes: list[Node]) -> list[Node]: else: loners.append((a,)) i += 1 - # может лучше проверять левый или правый if i == len(group) - 1: loners.append(group[i]) -- 2.54.0 From c4c5940f8d22a628bdc6c49a5d87e9acdc716f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 22:40:23 +0300 Subject: [PATCH 24/89] cosmetic changes --- cidr4_merger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 1bce248..d5622eb 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -72,7 +72,7 @@ def reduce_nodes(nodes: list[Node]) -> list[Node]: neighbours.append((a, b)) i += 2 else: - loners.append((a,)) + loners.append(a) i += 1 if i == len(group) - 1: loners.append(group[i]) -- 2.54.0 From 9d5c6b384a109a4d0c6b339fa1608bcd67e7734d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sat, 11 Jan 2025 23:59:29 +0300 Subject: [PATCH 25/89] add algorithm --- cidr4_merger.py | 87 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 12 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index d5622eb..6b802f4 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,4 +1,5 @@ import cProfile +import math from typing import Optional Node = tuple[int, int, int] @@ -46,7 +47,7 @@ def have_same_parent(a: Node, b: Node) -> bool: def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: max_mask_len = max(nodes, key=lambda x: x[1])[1] - return sort_nodes(filter(lambda x: x[1] == max_mask_len, nodes)) + return list(filter(lambda x: x[1] == max_mask_len, nodes)) def get_parent(a: Node, b: Node = None) -> Node: @@ -97,25 +98,54 @@ def reduce_nodes(nodes: list[Node]) -> list[Node]: def merge_nodes(nodes: list[Node], required_len: int) -> list[Node]: - pass + while len(nodes) > required_len: + nodes = reduce_nodes(nodes) + return nodes + + +def node_to_cidr4(node: Node) -> str: + ip_value, mask_len, _ = node + lst = [str(ip_value >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip = ".".join(lst) + return f"{ip}/{mask_len}" + + +def answer(nodes: list[Node], required_len: int) -> tuple[list[str], int]: + nodes = sort_nodes(nodes) + merged_nodes = merge_nodes(nodes, required_len) + + cidr4s = [] + sum_added_ips = 0 + for node in merged_nodes: + cidr4s.append(node_to_cidr4(node)) + sum_added_ips += node[2] + return cidr4s, sum_added_ips def main(): file = "cidr4.txt" - required_len = 15 + required_len = 20 data = get_data(file) nodes = data_to_nodes(data) - # for n in nodes: - # print(n) + cidr4s, sum_added_ips = answer(nodes, required_len) - merged_nodes = merge_nodes(nodes, required_len) + cidr4s_str = "\n".join(cidr4s) + print( + f"Исходный список длины {len(nodes)} сокращен до {len(cidr4s)}\n" + f"Количество добавленных ip адресов: {sum_added_ips:_}\n" + f"Список объединенных cidr4:\n" + f"{cidr4s_str}" + ) if __name__ == "__main__": assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) + assert node_to_cidr4((72256256, 24, 0)) == "4.78.139.0/24" + assert node_to_cidr4((0, 32, 10)) == "0.0.0.0/32" + bin_a = "10011000000000001000010000010000" assert len(bin_a) == 32 value_a = int(bin_a, 2) @@ -183,25 +213,21 @@ if __name__ == "__main__": assert reduce_nodes( [ - (0, 0, 0), (0, 2, 12), (1073741824, 2, 3), ] ) == [ - (0, 0, 0), (0, 1, 15), ] assert reduce_nodes( [ - (0, 0, 0), (0, 2, 12), (1073741824, 2, 3), (2147483648, 2, 1), (3221225472, 2, 2), ] ) == [ - (0, 0, 0), (2147483648, 1, 3), (0, 2, 12), (1073741824, 2, 3), @@ -226,5 +252,42 @@ if __name__ == "__main__": (0, 0, 12 + 2**30), ] - # main() - # cProfile.run("main()") + assert merge_nodes( + [ + (0, 2, 12), + (2147483648, 2, 1), + (3221225472, 2, 2), + ], + 2, + ) == [ + (2147483648, 1, 3), + (0, 2, 12), + ] + + assert merge_nodes( + [ + (0, 2, 12), + (2147483648, 2, 1), + (3221225472, 2, 2), + ], + 1, + ) == [(0, 0, 15 + 2**30)] + + assert answer( + [ + (0, 2, 0), + (2147483648, 2, 0), + (3221225472, 2, 12), + ], + 2, + ) == (["128.0.0.0/1", "0.0.0.0/2"], 12) + + assert answer( + [ + (0, 2, 0), + (2147483648, 2, 0), + ], + 1, + ) == (["0.0.0.0/0"], 2**31) + + cProfile.run("main()") -- 2.54.0 From 5ef04aefdfca1919fd8f257d3b31fc7f8c57f43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9F=D0=B0=D0=B2=D0=B5=D0=BB?= Date: Sun, 12 Jan 2025 00:00:33 +0300 Subject: [PATCH 26/89] cleanup --- cidr4_merger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 6b802f4..5216acf 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,5 +1,4 @@ import cProfile -import math from typing import Optional Node = tuple[int, int, int] -- 2.54.0 From 5f0353d574897f7153e42a23169822e9e33b3856 Mon Sep 17 00:00:00 2001 From: Fedor Lyanguzov Date: Sun, 12 Jan 2025 18:53:24 +0300 Subject: [PATCH 27/89] =?UTF-8?q?2025-01-12=20=D0=BE=D0=B1=D1=81=D1=83?= =?UTF-8?q?=D0=B6=D0=B4=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BA=D0=BE=D0=B4=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cidr4_merger.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 5216acf..30aae68 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,7 +1,8 @@ import cProfile from typing import Optional +from collections.abc import Iterator -Node = tuple[int, int, int] +Node = tuple[int, int, int] # только аннотация def get_data(input_file): @@ -13,31 +14,30 @@ def get_data(input_file): def cidr4_to_node(cidr4: str) -> Node: ip, mask_len = cidr4.strip().split("/") mask_len = int(mask_len) - added_ips_number = 0 a, b, c, d = list(map(int, ip.split("."))) ip_value = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 + added_ips_number = 0 return ip_value, mask_len, added_ips_number -def sort_nodes(nodes: list[Node]) -> list[Node]: +def sort_nodes(nodes: Iterator[Node]) -> list[Node]: return sorted(nodes, key=lambda x: (x[1], x[0])) -def data_to_nodes(data: list[str]) -> list[Node]: - return list(map(cidr4_to_node, data)) +def data_to_nodes(data: Iterator[str]) -> Iterator[Node]: + return map(cidr4_to_node, data) - -def get_mask(node: Node) -> int: - value, mask_len, _ = node - x = (2**mask_len - 1) << (32 - mask_len) - mask = value & x - return mask +def get_mask(ip, mask_len) -> int: + mask = ((1< Optional[int]: + a, b, _ = node if node[1] == 0: return None - return get_mask((node[0], node[1] - 1, node[2])) + return get_mask(a, b-1) def have_same_parent(a: Node, b: Node) -> bool: @@ -51,6 +51,7 @@ def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: def get_parent(a: Node, b: Node = None) -> Node: ip_value = get_parent_mask(a) + # обычно стоит вскрыть кортеж mask_len = a[1] - 1 added_ips_number = a[2] + 2 ** (32 - a[1]) if b: @@ -102,8 +103,7 @@ def merge_nodes(nodes: list[Node], required_len: int) -> list[Node]: return nodes -def node_to_cidr4(node: Node) -> str: - ip_value, mask_len, _ = node +def node_to_cidr4(ip_value, mask_len) -> str: lst = [str(ip_value >> (i << 3) & 0xFF) for i in reversed(range(4))] ip = ".".join(lst) return f"{ip}/{mask_len}" @@ -115,9 +115,9 @@ def answer(nodes: list[Node], required_len: int) -> tuple[list[str], int]: cidr4s = [] sum_added_ips = 0 - for node in merged_nodes: - cidr4s.append(node_to_cidr4(node)) - sum_added_ips += node[2] + for ip_value, mask_len, added_ips in merged_nodes: + cidr4s.append(node_to_cidr4(ip_value, mask_len)) + sum_added_ips += added_ips return cidr4s, sum_added_ips @@ -161,10 +161,10 @@ if __name__ == "__main__": assert len(bin_c) == 32 value_d = int(bin_d, 2) - assert get_mask((value_a, 5, 0)) == value_c - assert get_mask((value_b, 5, 0)) == value_c - assert get_mask((0, 1, 0)) == 0 - assert get_mask((0, 0, 0)) == 0 + assert get_mask(value_a, 5) == value_c + assert get_mask(value_b, 5) == value_c + assert get_mask(0, 1) == 0 + assert get_mask(0, 0) == 0 assert get_parent_mask((value_a, 6, 0)) == value_c assert get_parent_mask((value_b, 6, 0)) == value_c -- 2.54.0 From 048d4941e4d7f1bb6818f6d2daf8099ab74ce3a2 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 16 Jan 2025 12:41:52 +0300 Subject: [PATCH 28/89] format by black and isort --- cidr4_merger.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 30aae68..5eccab7 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,8 +1,8 @@ import cProfile -from typing import Optional from collections.abc import Iterator +from typing import Optional -Node = tuple[int, int, int] # только аннотация +Node = tuple[int, int, int] def get_data(input_file): @@ -27,8 +27,9 @@ def sort_nodes(nodes: Iterator[Node]) -> list[Node]: def data_to_nodes(data: Iterator[str]) -> Iterator[Node]: return map(cidr4_to_node, data) + def get_mask(ip, mask_len) -> int: - mask = ((1< Optional[int]: a, b, _ = node if node[1] == 0: return None - return get_mask(a, b-1) + return get_mask(a, b - 1) def have_same_parent(a: Node, b: Node) -> bool: -- 2.54.0 From d1afaace6852ad9fe102a91ee21ed51c913fbe3e Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 16 Jan 2025 12:42:18 +0300 Subject: [PATCH 29/89] fix tests --- cidr4_merger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 5eccab7..14967e1 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -143,8 +143,8 @@ if __name__ == "__main__": assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) - assert node_to_cidr4((72256256, 24, 0)) == "4.78.139.0/24" - assert node_to_cidr4((0, 32, 10)) == "0.0.0.0/32" + assert node_to_cidr4(72256256, 24) == "4.78.139.0/24" + assert node_to_cidr4(0, 32) == "0.0.0.0/32" bin_a = "10011000000000001000010000010000" assert len(bin_a) == 32 -- 2.54.0 From dd90ad624a4de67a37a551bd2260c6f7a4ca1e4c Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 16 Jan 2025 14:08:38 +0300 Subject: [PATCH 30/89] use tuple opening, use only necessary arguments in functions, delete data_to_nodes function --- cidr4_merger.py | 78 ++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 14967e1..a1d9278 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,5 +1,4 @@ import cProfile -from collections.abc import Iterator from typing import Optional Node = tuple[int, int, int] @@ -20,29 +19,26 @@ def cidr4_to_node(cidr4: str) -> Node: return ip_value, mask_len, added_ips_number -def sort_nodes(nodes: Iterator[Node]) -> list[Node]: +def sort_nodes(nodes: list[Node]) -> list[Node]: return sorted(nodes, key=lambda x: (x[1], x[0])) -def data_to_nodes(data: Iterator[str]) -> Iterator[Node]: - return map(cidr4_to_node, data) - - -def get_mask(ip, mask_len) -> int: +def get_net_addr(ip: int, mask_len: int) -> int: mask = ((1 << mask_len) - 1) << (32 - mask_len) - netaddr = ip & mask - return netaddr + net_addr = ip & mask + return net_addr -def get_parent_mask(node: Node) -> Optional[int]: - a, b, _ = node - if node[1] == 0: +def get_parent_mask(ip: int, mask_len: int) -> Optional[int]: + if mask_len == 0: return None - return get_mask(a, b - 1) + return get_net_addr(ip, mask_len - 1) -def have_same_parent(a: Node, b: Node) -> bool: - return a[1] == b[1] and get_parent_mask(a) == get_parent_mask(b) +def have_same_parent(ip_a, mask_len_a, ip_b, mask_len_b) -> bool: + return mask_len_a == mask_len_b and get_parent_mask( + ip_a, mask_len_a + ) == get_parent_mask(ip_b, mask_len_b) def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: @@ -50,15 +46,16 @@ def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: return list(filter(lambda x: x[1] == max_mask_len, nodes)) -def get_parent(a: Node, b: Node = None) -> Node: - ip_value = get_parent_mask(a) - # обычно стоит вскрыть кортеж - mask_len = a[1] - 1 - added_ips_number = a[2] + 2 ** (32 - a[1]) +def get_parent(a: Node, b: Optional[Node] = None) -> Node: + ip_a, mask_len_a, added_ips_number_a = a + ip = get_parent_mask(ip_a, mask_len_a) + mask_len = mask_len_a - 1 + added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) if b: - assert have_same_parent(a, b), "ноды должны быть соседями" - added_ips_number = a[2] + b[2] - return ip_value, mask_len, added_ips_number + ip_b, mask_len_b, added_ips_b = b + assert have_same_parent(ip_a, mask_len_a, ip_b, mask_len_b) + added_ips_number = added_ips_number_a + added_ips_b + return ip, mask_len, added_ips_number def reduce_nodes(nodes: list[Node]) -> list[Node]: @@ -68,9 +65,10 @@ def reduce_nodes(nodes: list[Node]) -> list[Node]: loners = [] i = 0 while i < len(group) - 1: - a = group[i] - b = group[i + 1] - if have_same_parent(a, b): + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _ = a + ip_b, mask_len_b, _ = b + if have_same_parent(ip_a, mask_len_a, ip_b, mask_len_b): neighbours.append((a, b)) i += 2 else: @@ -127,7 +125,7 @@ def main(): required_len = 20 data = get_data(file) - nodes = data_to_nodes(data) + nodes = list(map(cidr4_to_node, data)) cidr4s, sum_added_ips = answer(nodes, required_len) cidr4s_str = "\n".join(cidr4s) @@ -162,21 +160,21 @@ if __name__ == "__main__": assert len(bin_c) == 32 value_d = int(bin_d, 2) - assert get_mask(value_a, 5) == value_c - assert get_mask(value_b, 5) == value_c - assert get_mask(0, 1) == 0 - assert get_mask(0, 0) == 0 + assert get_net_addr(value_a, 5) == value_c + assert get_net_addr(value_b, 5) == value_c + assert get_net_addr(0, 1) == 0 + assert get_net_addr(0, 0) == 0 - assert get_parent_mask((value_a, 6, 0)) == value_c - assert get_parent_mask((value_b, 6, 0)) == value_c - assert get_parent_mask((0, 1, 0)) == 0 - assert get_parent_mask((0, 0, 0)) is None + assert get_parent_mask(value_a, 6) == value_c + assert get_parent_mask(value_b, 6) == value_c + assert get_parent_mask(0, 1) == 0 + assert get_parent_mask(0, 0) is None - assert have_same_parent((value_a, 6, 0), (value_b, 6, 0)) is True - assert have_same_parent((value_a, 6, 0), (value_b, 5, 0)) is False - assert have_same_parent((value_a, 6, 0), (value_d, 6, 0)) is False - assert have_same_parent((value_a, 6, 0), (0, 1, 0)) is False - assert have_same_parent((value_a, 6, 0), (0, 0, 0)) is False + assert have_same_parent(value_a, 6, value_b, 6) is True + assert have_same_parent(value_a, 6, value_b, 5) is False + assert have_same_parent(value_a, 6, value_d, 6) is False + assert have_same_parent(value_a, 6, 0, 1) is False + assert have_same_parent(value_a, 6, 0, 0) is False assert sort_nodes( [ -- 2.54.0 From a1dba6affb0f0cbe05b721fc25094d49080b6646 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 16 Jan 2025 15:12:30 +0300 Subject: [PATCH 31/89] move tests to a separate file --- cidr4_merger.py | 150 ----------------------------- tests/__init__.py | 0 tests/test_cidr4_merger.py | 189 +++++++++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 150 deletions(-) create mode 100644 tests/__init__.py diff --git a/cidr4_merger.py b/cidr4_merger.py index a1d9278..bd6d4b4 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -138,154 +138,4 @@ def main(): if __name__ == "__main__": - assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) - assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) - - assert node_to_cidr4(72256256, 24) == "4.78.139.0/24" - assert node_to_cidr4(0, 32) == "0.0.0.0/32" - - bin_a = "10011000000000001000010000010000" - assert len(bin_a) == 32 - value_a = int(bin_a, 2) - - bin_b = "10011100000000000000000000101011" - assert len(bin_b) == 32 - value_b = int(bin_b, 2) - - bin_c = "10011000000000000000000000000000" - assert len(bin_c) == 32 - value_c = int(bin_c, 2) - - bin_d = "11111100000000000000000000000000" - assert len(bin_c) == 32 - value_d = int(bin_d, 2) - - assert get_net_addr(value_a, 5) == value_c - assert get_net_addr(value_b, 5) == value_c - assert get_net_addr(0, 1) == 0 - assert get_net_addr(0, 0) == 0 - - assert get_parent_mask(value_a, 6) == value_c - assert get_parent_mask(value_b, 6) == value_c - assert get_parent_mask(0, 1) == 0 - assert get_parent_mask(0, 0) is None - - assert have_same_parent(value_a, 6, value_b, 6) is True - assert have_same_parent(value_a, 6, value_b, 5) is False - assert have_same_parent(value_a, 6, value_d, 6) is False - assert have_same_parent(value_a, 6, 0, 1) is False - assert have_same_parent(value_a, 6, 0, 0) is False - - assert sort_nodes( - [ - (401219072, 24, 0), - (2899902464, 19, 0), - (400657664, 24, 0), - (520969728, 23, 0), - ] - ) == [ - (2899902464, 19, 0), - (520969728, 23, 0), - (400657664, 24, 0), - (401219072, 24, 0), - ] - - assert get_group_with_max_mask_len( - [ - (2899902464, 19, 0), - (520969728, 23, 0), - (400657664, 24, 0), - (401219072, 24, 0), - ] - ) == [(400657664, 24, 0), (401219072, 24, 0)] - assert get_group_with_max_mask_len( - [ - (401219072, 24, 0), - (2899902464, 19, 0), - (520969728, 23, 0), - ] - ) == [(401219072, 24, 0)] - - assert get_parent((0, 2, 12), (1073741824, 2, 3)) == (0, 1, 15) - assert get_parent((2147483648, 2, 1), (3221225472, 2, 2)) == (2147483648, 1, 3) - - assert reduce_nodes( - [ - (0, 2, 12), - (1073741824, 2, 3), - ] - ) == [ - (0, 1, 15), - ] - - assert reduce_nodes( - [ - (0, 2, 12), - (1073741824, 2, 3), - (2147483648, 2, 1), - (3221225472, 2, 2), - ] - ) == [ - (2147483648, 1, 3), - (0, 2, 12), - (1073741824, 2, 3), - ] - - assert reduce_nodes( - [ - (0, 2, 12), - (2147483648, 1, 0), - ] - ) == [ - (0, 1, 12 + 2**30), - (2147483648, 1, 0), - ] - - assert reduce_nodes( - [ - (0, 1, 12 + 2**30), - (2147483648, 1, 0), - ] - ) == [ - (0, 0, 12 + 2**30), - ] - - assert merge_nodes( - [ - (0, 2, 12), - (2147483648, 2, 1), - (3221225472, 2, 2), - ], - 2, - ) == [ - (2147483648, 1, 3), - (0, 2, 12), - ] - - assert merge_nodes( - [ - (0, 2, 12), - (2147483648, 2, 1), - (3221225472, 2, 2), - ], - 1, - ) == [(0, 0, 15 + 2**30)] - - assert answer( - [ - (0, 2, 0), - (2147483648, 2, 0), - (3221225472, 2, 12), - ], - 2, - ) == (["128.0.0.0/1", "0.0.0.0/2"], 12) - - assert answer( - [ - (0, 2, 0), - (2147483648, 2, 0), - ], - 1, - ) == (["0.0.0.0/0"], 2**31) - cProfile.run("main()") diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 1f3b77f..d2e08bd 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -1,2 +1,191 @@ +from cidr4_merger import ( + answer, + cidr4_to_node, + get_group_with_max_mask_len, + get_net_addr, + get_parent, + get_parent_mask, + have_same_parent, + merge_nodes, + node_to_cidr4, + reduce_nodes, + sort_nodes, +) + + def test_true(): assert True + + +bin_a = "10011000000000001000010000010000" +assert len(bin_a) == 32 +value_a = int(bin_a, 2) + +bin_b = "10011100000000000000000000101011" +assert len(bin_b) == 32 +value_b = int(bin_b, 2) + +bin_c = "10011000000000000000000000000000" +assert len(bin_c) == 32 +value_c = int(bin_c, 2) + +bin_d = "11111100000000000000000000000000" +assert len(bin_c) == 32 +value_d = int(bin_d, 2) + + +def test_cidr4_to_node(): + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) + + +def test_node_to_cidr4(): + assert node_to_cidr4(72256256, 24) == "4.78.139.0/24" + assert node_to_cidr4(0, 32) == "0.0.0.0/32" + + +def test_get_net_addr(): + assert get_net_addr(value_a, 5) == value_c + assert get_net_addr(value_b, 5) == value_c + assert get_net_addr(0, 1) == 0 + assert get_net_addr(0, 0) == 0 + + +def test_get_parent_mask(): + assert get_parent_mask(value_a, 6) == value_c + assert get_parent_mask(value_b, 6) == value_c + assert get_parent_mask(0, 1) == 0 + assert get_parent_mask(0, 0) is None + + +def test_have_same_parent(): + assert have_same_parent(value_a, 6, value_b, 6) is True + assert have_same_parent(value_a, 6, value_b, 5) is False + assert have_same_parent(value_a, 6, value_d, 6) is False + assert have_same_parent(value_a, 6, 0, 1) is False + assert have_same_parent(value_a, 6, 0, 0) is False + + +def test_sort_nodes(): + assert sort_nodes( + [ + (401219072, 24, 0), + (2899902464, 19, 0), + (400657664, 24, 0), + (520969728, 23, 0), + ] + ) == [ + (2899902464, 19, 0), + (520969728, 23, 0), + (400657664, 24, 0), + (401219072, 24, 0), + ] + + +def test_get_group_with_max_mask_len(): + assert get_group_with_max_mask_len( + [ + (2899902464, 19, 0), + (520969728, 23, 0), + (400657664, 24, 0), + (401219072, 24, 0), + ] + ) == [(400657664, 24, 0), (401219072, 24, 0)] + + assert get_group_with_max_mask_len( + [ + (401219072, 24, 0), + (2899902464, 19, 0), + (520969728, 23, 0), + ] + ) == [(401219072, 24, 0)] + + +def test_get_parent(): + assert get_parent((0, 2, 12), (1073741824, 2, 3)) == (0, 1, 15) + assert get_parent((2147483648, 2, 1), (3221225472, 2, 2)) == (2147483648, 1, 3) + + +def test_reduce_nodes(): + assert reduce_nodes( + [ + (0, 2, 12), + (1073741824, 2, 3), + ] + ) == [ + (0, 1, 15), + ] + + assert reduce_nodes( + [ + (0, 2, 12), + (1073741824, 2, 3), + (2147483648, 2, 1), + (3221225472, 2, 2), + ] + ) == [ + (2147483648, 1, 3), + (0, 2, 12), + (1073741824, 2, 3), + ] + + assert reduce_nodes( + [ + (0, 2, 12), + (2147483648, 1, 0), + ] + ) == [ + (0, 1, 12 + 2**30), + (2147483648, 1, 0), + ] + + assert reduce_nodes( + [ + (0, 1, 12 + 2**30), + (2147483648, 1, 0), + ] + ) == [ + (0, 0, 12 + 2**30), + ] + + +def test_merge_nodes(): + assert merge_nodes( + [ + (0, 2, 12), + (2147483648, 2, 1), + (3221225472, 2, 2), + ], + 2, + ) == [ + (2147483648, 1, 3), + (0, 2, 12), + ] + + assert merge_nodes( + [ + (0, 2, 12), + (2147483648, 2, 1), + (3221225472, 2, 2), + ], + 1, + ) == [(0, 0, 15 + 2**30)] + + +def test_answer(): + assert answer( + [ + (0, 2, 0), + (2147483648, 2, 0), + (3221225472, 2, 12), + ], + 2, + ) == (["128.0.0.0/1", "0.0.0.0/2"], 12) + + assert answer( + [ + (0, 2, 0), + (2147483648, 2, 0), + ], + 1, + ) == (["0.0.0.0/0"], 2**31) -- 2.54.0 From edcce93d019435da2b4c6173efa3e6b76e0c1c1c Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 16 Jan 2025 15:18:08 +0300 Subject: [PATCH 32/89] rename test ips --- tests/test_cidr4_merger.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index d2e08bd..0d1e3d5 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -19,19 +19,19 @@ def test_true(): bin_a = "10011000000000001000010000010000" assert len(bin_a) == 32 -value_a = int(bin_a, 2) +ip_a = int(bin_a, 2) bin_b = "10011100000000000000000000101011" assert len(bin_b) == 32 -value_b = int(bin_b, 2) +ip_b = int(bin_b, 2) bin_c = "10011000000000000000000000000000" assert len(bin_c) == 32 -value_c = int(bin_c, 2) +ip_c = int(bin_c, 2) bin_d = "11111100000000000000000000000000" assert len(bin_c) == 32 -value_d = int(bin_d, 2) +ip_d = int(bin_d, 2) def test_cidr4_to_node(): @@ -45,25 +45,25 @@ def test_node_to_cidr4(): def test_get_net_addr(): - assert get_net_addr(value_a, 5) == value_c - assert get_net_addr(value_b, 5) == value_c + assert get_net_addr(ip_a, 5) == ip_c + assert get_net_addr(ip_b, 5) == ip_c assert get_net_addr(0, 1) == 0 assert get_net_addr(0, 0) == 0 def test_get_parent_mask(): - assert get_parent_mask(value_a, 6) == value_c - assert get_parent_mask(value_b, 6) == value_c + assert get_parent_mask(ip_a, 6) == ip_c + assert get_parent_mask(ip_b, 6) == ip_c assert get_parent_mask(0, 1) == 0 assert get_parent_mask(0, 0) is None def test_have_same_parent(): - assert have_same_parent(value_a, 6, value_b, 6) is True - assert have_same_parent(value_a, 6, value_b, 5) is False - assert have_same_parent(value_a, 6, value_d, 6) is False - assert have_same_parent(value_a, 6, 0, 1) is False - assert have_same_parent(value_a, 6, 0, 0) is False + assert have_same_parent(ip_a, 6, ip_b, 6) is True + assert have_same_parent(ip_a, 6, ip_b, 5) is False + assert have_same_parent(ip_a, 6, ip_d, 6) is False + assert have_same_parent(ip_a, 6, 0, 1) is False + assert have_same_parent(ip_a, 6, 0, 0) is False def test_sort_nodes(): -- 2.54.0 From b5a1498a51158290362c17f5dee25d90c35c285e Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 18 Jan 2025 20:15:21 +0300 Subject: [PATCH 33/89] add parent_ip info to Node tuple --- cidr4_merger.py | 67 +++++++------ tests/test_cidr4_merger.py | 190 +++++++++++++++++++++---------------- 2 files changed, 146 insertions(+), 111 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index bd6d4b4..4133171 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,7 +1,10 @@ import cProfile -from typing import Optional -Node = tuple[int, int, int] +Node = tuple[int, int, int, int] + + +class Cidr4MergerError(Exception): + pass def get_data(input_file): @@ -11,12 +14,13 @@ def get_data(input_file): def cidr4_to_node(cidr4: str) -> Node: - ip, mask_len = cidr4.strip().split("/") + ip_str, mask_len = cidr4.strip().split("/") mask_len = int(mask_len) - a, b, c, d = list(map(int, ip.split("."))) - ip_value = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 + a, b, c, d = list(map(int, ip_str.split("."))) + ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 added_ips_number = 0 - return ip_value, mask_len, added_ips_number + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip def sort_nodes(nodes: list[Node]) -> list[Node]: @@ -29,16 +33,14 @@ def get_net_addr(ip: int, mask_len: int) -> int: return net_addr -def get_parent_mask(ip: int, mask_len: int) -> Optional[int]: +def get_parent_ip(ip: int, mask_len: int) -> int: if mask_len == 0: - return None + raise Cidr4MergerError("The top of the tree has no parent!") return get_net_addr(ip, mask_len - 1) -def have_same_parent(ip_a, mask_len_a, ip_b, mask_len_b) -> bool: - return mask_len_a == mask_len_b and get_parent_mask( - ip_a, mask_len_a - ) == get_parent_mask(ip_b, mask_len_b) +def have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b) -> bool: + return mask_len_a == mask_len_b and parent_ip_a == parent_ip_b def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: @@ -46,16 +48,19 @@ def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: return list(filter(lambda x: x[1] == max_mask_len, nodes)) -def get_parent(a: Node, b: Optional[Node] = None) -> Node: - ip_a, mask_len_a, added_ips_number_a = a - ip = get_parent_mask(ip_a, mask_len_a) - mask_len = mask_len_a - 1 - added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) +def make_parent(a: Node, b: Node | None = None) -> Node: + ip_a, mask_len_a, added_ips_number_a, parent_ip_a = a if b: - ip_b, mask_len_b, added_ips_b = b - assert have_same_parent(ip_a, mask_len_a, ip_b, mask_len_b) + ip_b, mask_len_b, added_ips_b, parent_ip_b = b + if not have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + raise Cidr4MergerError("Nodes must be neighbors!") added_ips_number = added_ips_number_a + added_ips_b - return ip, mask_len, added_ips_number + else: + added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) + ip = parent_ip_a + mask_len = mask_len_a - 1 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip def reduce_nodes(nodes: list[Node]) -> list[Node]: @@ -66,9 +71,9 @@ def reduce_nodes(nodes: list[Node]) -> list[Node]: i = 0 while i < len(group) - 1: a, b = group[i], group[i + 1] - ip_a, mask_len_a, _ = a - ip_b, mask_len_b, _ = b - if have_same_parent(ip_a, mask_len_a, ip_b, mask_len_b): + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): neighbours.append((a, b)) i += 2 else: @@ -78,14 +83,14 @@ def reduce_nodes(nodes: list[Node]) -> list[Node]: loners.append(group[i]) if neighbours: - zipped = zip(neighbours, map(lambda x: get_parent(x[0], x[1]), neighbours)) + zipped = zip(neighbours, map(lambda x: make_parent(x[0], x[1]), neighbours)) min_zipped = min(zipped, key=lambda x: x[1][2]) (a, b), parent = min_zipped nodes.remove(a) nodes.remove(b) nodes.append(parent) elif loners: - zipped = zip(loners, map(get_parent, loners)) + zipped = zip(loners, map(make_parent, loners)) min_zipped = min(zipped, key=lambda x: x[1][2]) a, parent = min_zipped nodes.remove(a) @@ -102,10 +107,10 @@ def merge_nodes(nodes: list[Node], required_len: int) -> list[Node]: return nodes -def node_to_cidr4(ip_value, mask_len) -> str: - lst = [str(ip_value >> (i << 3) & 0xFF) for i in reversed(range(4))] - ip = ".".join(lst) - return f"{ip}/{mask_len}" +def make_cidr4(ip, mask_len) -> str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_str = ".".join(lst) + return f"{ip_str}/{mask_len}" def answer(nodes: list[Node], required_len: int) -> tuple[list[str], int]: @@ -114,8 +119,8 @@ def answer(nodes: list[Node], required_len: int) -> tuple[list[str], int]: cidr4s = [] sum_added_ips = 0 - for ip_value, mask_len, added_ips in merged_nodes: - cidr4s.append(node_to_cidr4(ip_value, mask_len)) + for ip_value, mask_len, added_ips, _ in merged_nodes: + cidr4s.append(make_cidr4(ip_value, mask_len)) sum_added_ips += added_ips return cidr4s, sum_added_ips diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 0d1e3d5..4035156 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -1,13 +1,16 @@ +import pytest + from cidr4_merger import ( + Cidr4MergerError, answer, cidr4_to_node, get_group_with_max_mask_len, get_net_addr, - get_parent, - get_parent_mask, + get_parent_ip, have_same_parent, + make_cidr4, + make_parent, merge_nodes, - node_to_cidr4, reduce_nodes, sort_nodes, ) @@ -35,13 +38,28 @@ ip_d = int(bin_d, 2) def test_cidr4_to_node(): - assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) - assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0, 72256000) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0, 0) + + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0, 401219072) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0, 2899902464) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0, 400657408) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0, 520969216) + + assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0, 0) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0, 0) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0, 2147483648) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0, 2147483648) -def test_node_to_cidr4(): - assert node_to_cidr4(72256256, 24) == "4.78.139.0/24" - assert node_to_cidr4(0, 32) == "0.0.0.0/32" +def test_make_cidr4(): + assert make_cidr4(72256256, 24) == "4.78.139.0/24" + assert make_cidr4(0, 32) == "0.0.0.0/32" + + assert make_cidr4(401219072, 24) == "23.234.30.0/24" + assert make_cidr4(2899902464, 19) == "172.217.0.0/19" + assert make_cidr4(400657664, 24) == "23.225.141.0/24" + assert make_cidr4(520969728, 23) == "31.13.94.0/23" def test_get_net_addr(): @@ -52,15 +70,19 @@ def test_get_net_addr(): def test_get_parent_mask(): - assert get_parent_mask(ip_a, 6) == ip_c - assert get_parent_mask(ip_b, 6) == ip_c - assert get_parent_mask(0, 1) == 0 - assert get_parent_mask(0, 0) is None + assert get_parent_ip(ip_a, 6) == ip_c + assert get_parent_ip(ip_b, 6) == ip_c + assert get_parent_ip(0, 1) == 0 + + with pytest.raises(Exception) as exc_info: + get_parent_ip(0, 0) + assert str(exc_info.value) == "The top of the tree has no parent!" + assert exc_info.type is Cidr4MergerError def test_have_same_parent(): - assert have_same_parent(ip_a, 6, ip_b, 6) is True - assert have_same_parent(ip_a, 6, ip_b, 5) is False + assert have_same_parent(ip_c, 6, ip_c, 6) is True + assert have_same_parent(ip_c, 6, ip_c, 5) is False assert have_same_parent(ip_a, 6, ip_d, 6) is False assert have_same_parent(ip_a, 6, 0, 1) is False assert have_same_parent(ip_a, 6, 0, 0) is False @@ -69,123 +91,131 @@ def test_have_same_parent(): def test_sort_nodes(): assert sort_nodes( [ - (401219072, 24, 0), - (2899902464, 19, 0), - (400657664, 24, 0), - (520969728, 23, 0), + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (400657664, 24, 0, 400657408), + (520969728, 23, 0, 520969216), ] ) == [ - (2899902464, 19, 0), - (520969728, 23, 0), - (400657664, 24, 0), - (401219072, 24, 0), + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), ] def test_get_group_with_max_mask_len(): assert get_group_with_max_mask_len( [ - (2899902464, 19, 0), - (520969728, 23, 0), - (400657664, 24, 0), - (401219072, 24, 0), + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), ] - ) == [(400657664, 24, 0), (401219072, 24, 0)] + ) == [(400657664, 24, 0, 400657408), (401219072, 24, 0, 401219072)] assert get_group_with_max_mask_len( [ - (401219072, 24, 0), - (2899902464, 19, 0), - (520969728, 23, 0), + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), ] - ) == [(401219072, 24, 0)] + ) == [(401219072, 24, 0, 401219072)] -def test_get_parent(): - assert get_parent((0, 2, 12), (1073741824, 2, 3)) == (0, 1, 15) - assert get_parent((2147483648, 2, 1), (3221225472, 2, 2)) == (2147483648, 1, 3) +def test_make_parent(): + assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) + assert make_parent( + (2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648) + ) == (2147483648, 1, 3, 0) + + with pytest.raises(Exception) as exc_info: + make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) + assert str(exc_info.value) == "Nodes must be neighbors!" + assert exc_info.type is Cidr4MergerError def test_reduce_nodes(): assert reduce_nodes( [ - (0, 2, 12), - (1073741824, 2, 3), + (0, 2, 12, 0), + (1073741824, 2, 3, 0), ] ) == [ - (0, 1, 15), + (0, 1, 15, 0), ] assert reduce_nodes( [ - (0, 2, 12), - (1073741824, 2, 3), - (2147483648, 2, 1), - (3221225472, 2, 2), + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), ] ) == [ - (2147483648, 1, 3), - (0, 2, 12), - (1073741824, 2, 3), + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + (1073741824, 2, 3, 0), ] assert reduce_nodes( [ - (0, 2, 12), - (2147483648, 1, 0), + (0, 2, 12, 0), + (2147483648, 1, 0, 0), ] ) == [ - (0, 1, 12 + 2**30), - (2147483648, 1, 0), + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), ] - assert reduce_nodes( - [ - (0, 1, 12 + 2**30), - (2147483648, 1, 0), - ] - ) == [ - (0, 0, 12 + 2**30), - ] + with pytest.raises(Exception) as exc_info: + reduce_nodes( + [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" def test_merge_nodes(): assert merge_nodes( [ - (0, 2, 12), - (2147483648, 2, 1), - (3221225472, 2, 2), + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), ], 2, ) == [ - (2147483648, 1, 3), - (0, 2, 12), + (2147483648, 1, 3, 0), + (0, 2, 12, 0), ] - assert merge_nodes( - [ - (0, 2, 12), - (2147483648, 2, 1), - (3221225472, 2, 2), - ], - 1, - ) == [(0, 0, 15 + 2**30)] + with pytest.raises(Exception) as exc_info: + merge_nodes( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" def test_answer(): assert answer( [ - (0, 2, 0), - (2147483648, 2, 0), - (3221225472, 2, 12), + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), ], 2, - ) == (["128.0.0.0/1", "0.0.0.0/2"], 12) + ) == (["128.0.0.0/1", "0.0.0.0/2"], 15) - assert answer( - [ - (0, 2, 0), - (2147483648, 2, 0), - ], - 1, - ) == (["0.0.0.0/0"], 2**31) + with pytest.raises(Exception) as exc_info: + answer([(0, 2, 0, 0), (2147483648, 2, 0, 2147483648)], 1) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" -- 2.54.0 From af91a066a13a31617a649718f3dafe054845486f Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 18 Jan 2025 20:24:04 +0300 Subject: [PATCH 34/89] rename ip_str to ip_address --- cidr4_merger.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 4133171..a25982a 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -14,9 +14,9 @@ def get_data(input_file): def cidr4_to_node(cidr4: str) -> Node: - ip_str, mask_len = cidr4.strip().split("/") + ip_address, mask_len = cidr4.strip().split("/") mask_len = int(mask_len) - a, b, c, d = list(map(int, ip_str.split("."))) + a, b, c, d = list(map(int, ip_address.split("."))) ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 added_ips_number = 0 parent_ip = get_parent_ip(ip, mask_len) @@ -109,8 +109,8 @@ def merge_nodes(nodes: list[Node], required_len: int) -> list[Node]: def make_cidr4(ip, mask_len) -> str: lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] - ip_str = ".".join(lst) - return f"{ip_str}/{mask_len}" + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}" def answer(nodes: list[Node], required_len: int) -> tuple[list[str], int]: -- 2.54.0 From feda229c78b1749eb03e54d34ee813e95f7ce01b Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 18 Jan 2025 23:17:47 +0300 Subject: [PATCH 35/89] add non-working recursive merge algorithm --- cidr4_merger.py | 85 +++++++++++++++++++++++++++++++++----- tests/test_cidr4_merger.py | 22 +++------- 2 files changed, 80 insertions(+), 27 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index a25982a..b1fd20f 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,4 +1,5 @@ import cProfile +from collections import defaultdict Node = tuple[int, int, int, int] @@ -113,16 +114,71 @@ def make_cidr4(ip, mask_len) -> str: return f"{ip_address}/{mask_len}" -def answer(nodes: list[Node], required_len: int) -> tuple[list[str], int]: - nodes = sort_nodes(nodes) - merged_nodes = merge_nodes(nodes, required_len) +def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: + # find single whose parent has the least added addresses + min_single, min_parent = singles[0], make_parent(singles[0]) + for node in singles[1:]: + parent = make_parent(node) + if parent[2] < min_parent[2]: + min_single, min_parent = node, parent - cidr4s = [] - sum_added_ips = 0 - for ip_value, mask_len, added_ips, _ in merged_nodes: - cidr4s.append(make_cidr4(ip_value, mask_len)) - sum_added_ips += added_ips - return cidr4s, sum_added_ips + new_nodes = [x for x in nodes] + new_nodes.remove(min_single) + new_nodes.append(min_parent) + new_nodes = sort_nodes(new_nodes) + return new_nodes + + +def merge_neighbors( + nodes: list[Node], neighbours: list[tuple[Node, Node]] +) -> list[Node]: + new_nodes = [x for x in nodes] + for a, b in neighbours: + parent = make_parent(a, b) + new_nodes.remove(a) + new_nodes.remove(b) + new_nodes.append(parent) + return sort_nodes(nodes) + + +def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: + neighbours = [] + singles = [] + for group in groups.values(): + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + singles.append(a) + i += 1 + if i == len(group) - 1: + singles.append(group[i]) + return neighbours, singles + + +def make_groups(nodes: list[Node]) -> defaultdict: + groups = defaultdict(list) + for n in nodes: + groups[n[1]].append(n) + return groups + + +def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: + if len(nodes) <= required_len: + return nodes + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + print(f"{len(nodes)=} {len(singles)=} {len(neighbours)=}") + if neighbours: + new_nodes = merge_neighbors(nodes, neighbours) + return merge_nodes_recursion(new_nodes, required_len) + new_nodes = lift_lonely_node(nodes, singles) + return merge_nodes_recursion(new_nodes, required_len) def main(): @@ -131,7 +187,16 @@ def main(): data = get_data(file) nodes = list(map(cidr4_to_node, data)) - cidr4s, sum_added_ips = answer(nodes, required_len) + + nodes = sort_nodes(nodes) + # merged_nodes = merge_nodes(nodes, required_len) + merged_nodes = merge_nodes_recursion(nodes, required_len) + + cidr4s = [] + sum_added_ips = 0 + for ip_value, mask_len, added_ips, _ in merged_nodes: + cidr4s.append(make_cidr4(ip_value, mask_len)) + sum_added_ips += added_ips cidr4s_str = "\n".join(cidr4s) print( diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 4035156..0be9164 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -2,15 +2,19 @@ import pytest from cidr4_merger import ( Cidr4MergerError, - answer, cidr4_to_node, + find_neighbours_singles, get_group_with_max_mask_len, get_net_addr, get_parent_ip, have_same_parent, + lift_lonely_node, make_cidr4, + make_groups, make_parent, + merge_neighbors, merge_nodes, + merge_nodes_recursion, reduce_nodes, sort_nodes, ) @@ -203,19 +207,3 @@ def test_merge_nodes(): ) assert exc_info.type is Cidr4MergerError assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_answer(): - assert answer( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 2, - ) == (["128.0.0.0/1", "0.0.0.0/2"], 15) - - with pytest.raises(Exception) as exc_info: - answer([(0, 2, 0, 0), (2147483648, 2, 0, 2147483648)], 1) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" -- 2.54.0 From a42f459db4e1fff7819ad68c8e1cb9e25a4ec33b Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 18 Jan 2025 23:19:04 +0300 Subject: [PATCH 36/89] add test make_groups --- tests/test_cidr4_merger.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 0be9164..58e89d4 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -207,3 +207,40 @@ def test_merge_nodes(): ) assert exc_info.type is Cidr4MergerError assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_make_groups(): + groups = make_groups( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + ) + assert dict(groups) == { + 2: [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + } + + +# def test_merge_nodes_recursion(): +# assert merge_nodes( +# [ +# (0, 2, 12, 0), +# (2147483648, 2, 1, 2147483648), +# (3221225472, 2, 2, 2147483648), +# ], +# 2, +# ) == merge_nodes_recursion( +# [ +# (0, 2, 12, 0), +# (2147483648, 2, 1, 2147483648), +# (3221225472, 2, 2, 2147483648), +# ], +# 2, +# ) -- 2.54.0 From c56d09cfd90481266019e1d9ed9effcd2bc22443 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 18 Jan 2025 23:52:28 +0300 Subject: [PATCH 37/89] extended the make_groups function test --- tests/test_cidr4_merger.py | 56 ++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 58e89d4..18af387 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -1,23 +1,11 @@ import pytest -from cidr4_merger import ( - Cidr4MergerError, - cidr4_to_node, - find_neighbours_singles, - get_group_with_max_mask_len, - get_net_addr, - get_parent_ip, - have_same_parent, - lift_lonely_node, - make_cidr4, - make_groups, - make_parent, - merge_neighbors, - merge_nodes, - merge_nodes_recursion, - reduce_nodes, - sort_nodes, -) +from cidr4_merger import (Cidr4MergerError, cidr4_to_node, + find_neighbours_singles, get_group_with_max_mask_len, + get_net_addr, get_parent_ip, have_same_parent, + lift_lonely_node, make_cidr4, make_groups, + make_parent, merge_neighbors, merge_nodes, + merge_nodes_recursion, reduce_nodes, sort_nodes) def test_true(): @@ -210,21 +198,23 @@ def test_merge_nodes(): def test_make_groups(): - groups = make_groups( - [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ] - ) - assert dict(groups) == { - 2: [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ] + nodes = [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + (520963072, 22, 0, 520962048), + (1089054720, 22, 0, 1089054720), + (2899902464, 19, 0, 2899902464), + (2915221504, 19, 0, 2915221504), + ] + assert dict(make_groups(nodes)) == { + 20: [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + ], + 22: [(520963072, 22, 0, 520962048), (1089054720, 22, 0, 1089054720)], + 19: [(2899902464, 19, 0, 2899902464), (2915221504, 19, 0, 2915221504)], } -- 2.54.0 From 95fbdff75fcc192756df540a6312812b7ae057e6 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 18 Jan 2025 23:54:00 +0300 Subject: [PATCH 38/89] add find_neighbours_singles function test --- tests/test_cidr4_merger.py | 62 ++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 18af387..3389d7f 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -1,11 +1,23 @@ import pytest -from cidr4_merger import (Cidr4MergerError, cidr4_to_node, - find_neighbours_singles, get_group_with_max_mask_len, - get_net_addr, get_parent_ip, have_same_parent, - lift_lonely_node, make_cidr4, make_groups, - make_parent, merge_neighbors, merge_nodes, - merge_nodes_recursion, reduce_nodes, sort_nodes) +from cidr4_merger import ( + Cidr4MergerError, + cidr4_to_node, + find_neighbours_singles, + get_group_with_max_mask_len, + get_net_addr, + get_parent_ip, + have_same_parent, + lift_lonely_node, + make_cidr4, + make_groups, + make_parent, + merge_neighbors, + merge_nodes, + merge_nodes_recursion, + reduce_nodes, + sort_nodes, +) def test_true(): @@ -218,6 +230,44 @@ def test_make_groups(): } +def test_find_neighbours_singles(): + nodes_with_neighbours = [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + groups_with_neighbours = make_groups(nodes_with_neighbours) + assert find_neighbours_singles(groups_with_neighbours) == ( + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + [], + ) + + nodes_with_singles = [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + ] + groups_with_singles = make_groups(nodes_with_singles) + assert find_neighbours_singles(groups_with_singles) == ( + [], + [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], + ) + + nodes_with_neighbours_n_singles = [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + ] + groups_with_singles = make_groups(nodes_with_neighbours_n_singles) + assert find_neighbours_singles(groups_with_singles) == ( + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + [(2147483648, 2, 1, 2147483648)], + ) + + # def test_merge_nodes_recursion(): # assert merge_nodes( # [ -- 2.54.0 From 58c2b5cbe05b56287738f7fbae0d743c4ccd8f78 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 00:05:21 +0300 Subject: [PATCH 39/89] add fixtures for nodes and groups --- tests/test_cidr4_merger.py | 66 ++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 3389d7f..efb0925 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -230,15 +230,54 @@ def test_make_groups(): } -def test_find_neighbours_singles(): - nodes_with_neighbours = [ +@pytest.fixture +def nodes_only_neighbours(): + return [ (0, 2, 12, 0), (1073741824, 2, 3, 0), (2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648), ] - groups_with_neighbours = make_groups(nodes_with_neighbours) - assert find_neighbours_singles(groups_with_neighbours) == ( + + +@pytest.fixture +def nodes_only_singles(): + return [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def nodes_with_neighbours_n_singles(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def groups_only_neighbours(nodes_only_neighbours): + return make_groups(nodes_only_neighbours) + + +@pytest.fixture +def groups_only_singles(nodes_only_singles): + return make_groups(nodes_only_singles) + + +@pytest.fixture +def groups_with_neighbours_n_singles(nodes_with_neighbours_n_singles): + return make_groups(nodes_with_neighbours_n_singles) + + +def test_find_neighbours_singles( + groups_only_neighbours, + groups_only_singles, + groups_with_neighbours_n_singles, +): + assert find_neighbours_singles(groups_only_neighbours) == ( [ ((0, 2, 12, 0), (1073741824, 2, 3, 0)), ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), @@ -246,28 +285,21 @@ def test_find_neighbours_singles(): [], ) - nodes_with_singles = [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - ] - groups_with_singles = make_groups(nodes_with_singles) - assert find_neighbours_singles(groups_with_singles) == ( + assert find_neighbours_singles(groups_only_singles) == ( [], [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], ) - nodes_with_neighbours_n_singles = [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - ] - groups_with_singles = make_groups(nodes_with_neighbours_n_singles) - assert find_neighbours_singles(groups_with_singles) == ( + assert find_neighbours_singles(groups_with_neighbours_n_singles) == ( [((0, 2, 12, 0), (1073741824, 2, 3, 0))], [(2147483648, 2, 1, 2147483648)], ) +def test_merge_neighbors(): + assert True + + # def test_merge_nodes_recursion(): # assert merge_nodes( # [ -- 2.54.0 From 71b5525c32dd77b9e3b9f3ff5a91defcf85e1d25 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 00:19:04 +0300 Subject: [PATCH 40/89] fix merge_neighbors and add merge_neighbors tests --- cidr4_merger.py | 2 +- tests/test_cidr4_merger.py | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index b1fd20f..e4b9575 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -138,7 +138,7 @@ def merge_neighbors( new_nodes.remove(a) new_nodes.remove(b) new_nodes.append(parent) - return sort_nodes(nodes) + return sort_nodes(new_nodes) def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index efb0925..12b8858 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -296,8 +296,29 @@ def test_find_neighbours_singles( ) -def test_merge_neighbors(): - assert True +def test_merge_neighbors__only_neighbours(nodes_only_neighbours): + new_nodes = merge_neighbors( + nodes_only_neighbours, + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 1, 3, 0), + ] + + +def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): + new_nodes = merge_neighbors( + nodes_with_neighbours_n_singles, + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 2, 1, 2147483648), + ] # def test_merge_nodes_recursion(): -- 2.54.0 From 724e8f8471218cc9f95748bd2a852e671c1d6f85 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 00:27:42 +0300 Subject: [PATCH 41/89] add lift_lonely_node test --- tests/test_cidr4_merger.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 12b8858..79265b2 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -321,6 +321,12 @@ def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): ] +def test_lift_lonely_node(nodes_only_singles): + singles = [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)] + new_nodes = lift_lonely_node(nodes_only_singles, singles) + assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] + + # def test_merge_nodes_recursion(): # assert merge_nodes( # [ -- 2.54.0 From c5e20446bed7301221d717afae28c268dca14c8e Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 00:29:54 +0300 Subject: [PATCH 42/89] add merge_nodes_recursion test --- tests/test_cidr4_merger.py | 40 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 79265b2..1e64e45 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -327,19 +327,27 @@ def test_lift_lonely_node(nodes_only_singles): assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] -# def test_merge_nodes_recursion(): -# assert merge_nodes( -# [ -# (0, 2, 12, 0), -# (2147483648, 2, 1, 2147483648), -# (3221225472, 2, 2, 2147483648), -# ], -# 2, -# ) == merge_nodes_recursion( -# [ -# (0, 2, 12, 0), -# (2147483648, 2, 1, 2147483648), -# (3221225472, 2, 2, 2147483648), -# ], -# 2, -# ) +def test_merge_nodes_recursion(): + assert merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" -- 2.54.0 From 1d34eb92bd8184716ccd671f2d94a14cb04c77f9 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 00:42:13 +0300 Subject: [PATCH 43/89] add working recursive merge algorithm --- cidr4_merger.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index e4b9575..6a214a4 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,6 +1,9 @@ import cProfile +import sys from collections import defaultdict +sys.setrecursionlimit(10_000) + Node = tuple[int, int, int, int] @@ -173,7 +176,6 @@ def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: return nodes groups = make_groups(nodes) neighbours, singles = find_neighbours_singles(groups) - print(f"{len(nodes)=} {len(singles)=} {len(neighbours)=}") if neighbours: new_nodes = merge_neighbors(nodes, neighbours) return merge_nodes_recursion(new_nodes, required_len) -- 2.54.0 From 8ce8ac6e1ac824624c15699af4da567117ea33c5 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 00:46:00 +0300 Subject: [PATCH 44/89] rename prev merge_nodes func to merge_nodes_deprecated --- cidr4_merger.py | 2 +- tests/test_cidr4_merger.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 6a214a4..8492b09 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -105,7 +105,7 @@ def reduce_nodes(nodes: list[Node]) -> list[Node]: return sort_nodes(nodes) -def merge_nodes(nodes: list[Node], required_len: int) -> list[Node]: +def merge_nodes_deprecated(nodes: list[Node], required_len: int) -> list[Node]: while len(nodes) > required_len: nodes = reduce_nodes(nodes) return nodes diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 1e64e45..5053299 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -13,7 +13,7 @@ from cidr4_merger import ( make_groups, make_parent, merge_neighbors, - merge_nodes, + merge_nodes_deprecated, merge_nodes_recursion, reduce_nodes, sort_nodes, @@ -183,8 +183,8 @@ def test_reduce_nodes(): assert str(exc_info.value) == "The top of the tree has no parent!" -def test_merge_nodes(): - assert merge_nodes( +def test_merge_nodes_deprecated(): + assert merge_nodes_deprecated( [ (0, 2, 12, 0), (2147483648, 2, 1, 2147483648), @@ -197,7 +197,7 @@ def test_merge_nodes(): ] with pytest.raises(Exception) as exc_info: - merge_nodes( + merge_nodes_deprecated( [ (0, 2, 12, 0), (2147483648, 2, 1, 2147483648), -- 2.54.0 From 39e99d24b6a3d277b82b9aef067c28fd9ad1bf52 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 01:00:01 +0300 Subject: [PATCH 45/89] add merge_nodes_cycle func --- cidr4_merger.py | 19 +++++++++++++++++-- tests/test_cidr4_merger.py | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 8492b09..d7306c7 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -183,6 +183,20 @@ def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: return merge_nodes_recursion(new_nodes, required_len) +def merge_nodes_cycle(nodes_to_merge: list[Node], required_len: int) -> list[Node]: + nodes = [x for x in nodes_to_merge] + while not len(nodes) <= required_len: + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + nodes = merge_neighbors(nodes, neighbours) + elif singles: + nodes = lift_lonely_node(nodes, singles) + else: + raise Cidr4MergerError("Invalid case!") + return nodes + + def main(): file = "cidr4.txt" required_len = 20 @@ -191,8 +205,9 @@ def main(): nodes = list(map(cidr4_to_node, data)) nodes = sort_nodes(nodes) - # merged_nodes = merge_nodes(nodes, required_len) - merged_nodes = merge_nodes_recursion(nodes, required_len) + # merged_nodes = merge_nodes_deprecated(nodes, required_len) + # merged_nodes = merge_nodes_recursion(nodes, required_len) + merged_nodes = merge_nodes_cycle(nodes, required_len) cidr4s = [] sum_added_ips = 0 diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 5053299..24a56a0 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -15,6 +15,7 @@ from cidr4_merger import ( merge_neighbors, merge_nodes_deprecated, merge_nodes_recursion, + merge_nodes_cycle, reduce_nodes, sort_nodes, ) @@ -351,3 +352,29 @@ def test_merge_nodes_recursion(): ) assert exc_info.type is Cidr4MergerError assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_cycle(): + assert merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" -- 2.54.0 From 7600814772c19847f632fd298c1c5be973cfeebc Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 01:24:08 +0300 Subject: [PATCH 46/89] add test merge_nodes recursion vs cycle --- tests/cidr4_data_for_tests.py | 488 ++++++++++++++++++++++++++++++++++ tests/test_cidr4_merger.py | 17 +- 2 files changed, 504 insertions(+), 1 deletion(-) create mode 100644 tests/cidr4_data_for_tests.py diff --git a/tests/cidr4_data_for_tests.py b/tests/cidr4_data_for_tests.py new file mode 100644 index 0000000..1e5bf85 --- /dev/null +++ b/tests/cidr4_data_for_tests.py @@ -0,0 +1,488 @@ +test_cidr4_data = """ +4.78.139.0/24 +23.101.24.0/24 +23.202.231.0/24 +23.217.138.0/24 +23.225.141.0/24 +23.234.30.0/24 +31.13.64.0/24 +31.13.67.0/24 +31.13.68.0/22 +31.13.73.0/24 +31.13.75.0/24 +31.13.76.0/24 +31.13.80.0/21 +31.13.88.0/24 +31.13.90.0/23 +31.13.92.0/24 +31.13.94.0/23 +31.13.96.0/24 +31.13.106.0/24 +31.13.112.0/24 +37.152.2.0/24 +38.121.72.0/24 +39.109.122.0/24 +43.226.16.0/24 +43.245.104.0/24 +45.54.28.0/24 +45.77.186.0/24 +45.114.11.0/24 +45.253.131.0/24 +46.61.154.0/24 +46.134.216.0/24 +47.88.58.0/24 +49.231.55.0/24 +50.23.209.0/24 +50.87.93.0/24 +50.117.117.0/24 +52.58.1.0/24 +52.175.9.0/24 +54.89.135.0/24 +54.144.128.0/24 +54.234.18.0/24 +58.63.233.0/24 +59.18.44.0/24 +59.18.46.0/24 +59.24.3.0/24 +59.188.250.0/24 +61.91.8.0/24 +61.205.119.0/24 +62.0.80.0/24 +64.13.192.0/24 +64.53.242.0/24 +64.233.160.0/21 +64.233.168.0/22 +64.233.176.0/20 +65.49.26.0/24 +65.49.68.0/24 +66.102.1.0/24 +66.220.146.0/23 +66.220.148.0/23 +67.15.100.0/24 +67.15.129.0/24 +67.228.102.0/24 +67.228.235.0/24 +67.230.169.0/24 +69.30.25.0/24 +69.50.221.0/24 +69.63.176.0/24 +69.63.178.0/24 +69.63.180.0/23 +69.63.184.0/24 +69.63.186.0/23 +69.63.190.0/24 +69.162.134.0/24 +69.171.224.0/24 +69.171.227.0/24 +69.171.228.0/23 +69.171.234.0/24 +69.171.242.0/24 +69.171.247.0/24 +69.197.153.0/24 +74.86.3.0/24 +74.86.12.0/24 +74.86.17.0/24 +74.86.118.0/24 +74.86.142.0/24 +74.86.151.0/24 +74.86.226.0/24 +74.86.228.0/24 +74.125.1.0/24 +74.125.2.0/24 +74.125.8.0/24 +74.125.10.0/23 +74.125.13.0/24 +74.125.20.0/23 +74.125.23.0/24 +74.125.24.0/24 +74.125.26.0/24 +74.125.28.0/24 +74.125.31.0/24 +74.125.68.0/22 +74.125.90.0/24 +74.125.96.0/24 +74.125.100.0/24 +74.125.102.0/23 +74.125.104.0/24 +74.125.106.0/24 +74.125.110.0/23 +74.125.124.0/24 +74.125.126.0/23 +74.125.128.0/20 +74.125.153.0/24 +74.125.154.0/23 +74.125.159.0/24 +74.125.164.0/24 +74.125.170.0/24 +74.125.172.0/23 +74.125.192.0/23 +74.125.195.0/24 +74.125.196.0/22 +74.125.200.0/22 +74.125.204.0/23 +74.125.206.0/24 +75.126.33.0/24 +75.126.115.0/24 +75.126.124.0/24 +75.126.135.0/24 +75.126.150.0/24 +75.126.164.0/24 +77.37.252.0/24 +77.120.15.0/24 +80.87.199.0/24 +81.23.20.0/24 +81.23.23.0/24 +81.192.13.0/24 +81.192.191.0/24 +88.191.249.0/24 +90.201.124.0/24 +92.87.232.0/24 +93.179.102.0/24 +94.24.232.0/24 +94.31.189.0/24 +95.59.170.0/24 +96.44.137.0/24 +98.159.108.0/24 +103.39.76.0/24 +103.42.176.0/24 +103.56.16.0/24 +103.73.161.0/24 +103.97.3.0/24 +103.97.176.0/24 +103.200.30.0/23 +103.214.168.0/24 +103.226.246.0/24 +103.228.130.0/24 +103.230.123.0/24 +103.240.180.0/24 +103.240.182.0/24 +103.246.246.0/24 +103.252.114.0/23 +104.16.251.0/24 +104.16.252.0/24 +104.23.124.0/23 +104.31.142.0/24 +104.244.43.0/24 +104.244.45.0/24 +104.244.46.0/24 +107.181.166.0/24 +108.160.161.0/24 +108.160.162.0/23 +108.160.165.0/24 +108.160.166.0/23 +108.160.169.0/24 +108.160.170.0/24 +108.160.172.0/23 +108.177.8.0/21 +108.177.96.0/23 +108.177.98.0/24 +108.177.103.0/24 +108.177.104.0/24 +108.177.111.0/24 +108.177.112.0/24 +108.177.119.0/24 +108.177.120.0/22 +108.177.125.0/24 +108.177.126.0/23 +109.224.41.0/24 +110.164.8.0/24 +111.243.214.0/24 +113.108.239.0/24 +113.171.242.0/24 +114.4.7.0/24 +114.43.24.0/24 +115.126.100.0/24 +116.89.243.0/24 +118.98.30.0/24 +118.98.36.0/24 +118.98.106.0/24 +118.107.180.0/24 +118.184.26.0/24 +118.184.78.0/24 +118.193.202.0/24 +118.193.240.0/24 +119.28.87.0/24 +120.232.233.0/24 +120.232.234.0/24 +120.233.71.0/24 +120.253.250.0/24 +120.253.253.0/24 +120.253.255.0/24 +121.78.42.0/24 +122.10.85.0/24 +122.154.76.0/24 +122.248.226.0/24 +122.252.245.0/24 +124.11.210.0/24 +128.121.146.0/24 +128.121.243.0/24 +128.242.240.0/24 +128.242.245.0/24 +128.242.250.0/24 +130.211.15.0/24 +142.250.0.0/23 +142.250.4.0/24 +142.250.8.0/22 +142.250.12.0/23 +142.250.27.0/24 +142.250.28.0/24 +142.250.30.0/23 +142.250.64.0/20 +142.250.80.0/23 +142.250.96.0/21 +142.250.105.0/24 +142.250.107.0/24 +142.250.110.0/23 +142.250.112.0/22 +142.250.123.0/24 +142.250.125.0/24 +142.250.126.0/24 +142.250.128.0/24 +142.250.136.0/24 +142.250.138.0/24 +142.250.141.0/24 +142.250.142.0/24 +142.250.145.0/24 +142.250.147.0/24 +142.250.148.0/23 +142.250.150.0/24 +142.250.152.0/23 +142.250.157.0/24 +142.250.158.0/23 +142.250.176.0/20 +142.250.192.0/21 +142.250.200.0/23 +142.250.203.0/24 +142.250.204.0/22 +142.250.217.0/24 +142.250.218.0/23 +142.251.0.0/23 +142.251.2.0/24 +142.251.4.0/23 +142.251.6.0/24 +142.251.8.0/23 +142.251.10.0/24 +142.251.12.0/24 +142.251.15.0/24 +142.251.16.0/24 +142.251.18.0/24 +142.251.31.0/24 +142.251.32.0/22 +142.251.36.0/23 +142.251.39.0/24 +142.251.40.0/22 +142.251.45.0/24 +142.251.46.0/23 +142.251.107.0/24 +142.251.111.0/24 +142.251.112.0/24 +142.251.116.0/23 +142.251.120.0/24 +142.251.128.0/23 +142.251.130.0/24 +142.251.132.0/22 +142.251.140.0/22 +142.251.161.0/24 +142.251.162.0/23 +142.251.164.0/22 +142.251.168.0/24 +142.251.170.0/23 +142.251.172.0/22 +142.251.176.0/22 +142.251.180.0/24 +142.251.182.0/23 +142.251.184.0/22 +142.251.188.0/24 +142.251.208.0/23 +142.251.211.0/24 +142.251.214.0/23 +142.251.216.0/24 +142.251.218.0/24 +142.251.220.0/22 +145.255.14.0/24 +148.163.48.0/24 +150.107.3.0/24 +154.0.29.0/24 +154.83.14.0/23 +154.85.102.0/24 +154.92.16.0/24 +156.233.67.0/24 +157.240.0.0/22 +157.240.6.0/23 +157.240.8.0/22 +157.240.12.0/23 +157.240.15.0/24 +157.240.16.0/23 +157.240.18.0/24 +157.240.20.0/23 +159.65.107.0/24 +159.106.121.0/24 +159.138.20.0/24 +162.125.1.0/24 +162.125.2.0/24 +162.125.6.0/23 +162.125.8.0/24 +162.125.17.0/24 +162.125.18.0/24 +162.125.32.0/24 +162.125.34.0/24 +162.125.80.0/24 +162.125.82.0/23 +162.220.12.0/24 +168.143.162.0/24 +168.143.171.0/24 +172.217.0.0/19 +172.217.129.0/24 +172.217.130.0/24 +172.217.133.0/24 +172.217.135.0/24 +172.217.160.0/20 +172.217.192.0/22 +172.217.197.0/24 +172.217.203.0/24 +172.217.204.0/24 +172.217.212.0/24 +172.217.214.0/23 +172.217.218.0/23 +172.217.222.0/24 +172.253.58.0/24 +172.253.62.0/23 +172.253.112.0/21 +172.253.120.0/24 +172.253.122.0/23 +172.253.124.0/22 +173.194.4.0/23 +173.194.12.0/24 +173.194.22.0/24 +173.194.28.0/23 +173.194.31.0/24 +173.194.49.0/24 +173.194.51.0/24 +173.194.54.0/23 +173.194.59.0/24 +173.194.65.0/24 +173.194.66.0/23 +173.194.68.0/23 +173.194.70.0/24 +173.194.73.0/24 +173.194.74.0/24 +173.194.76.0/22 +173.194.135.0/24 +173.194.150.0/24 +173.194.154.0/24 +173.194.161.0/24 +173.194.162.0/23 +173.194.164.0/24 +173.194.166.0/23 +173.194.174.0/23 +173.194.178.0/24 +173.194.182.0/23 +173.194.184.0/23 +173.194.187.0/24 +173.194.188.0/24 +173.194.190.0/23 +173.194.192.0/19 +173.208.182.0/24 +173.231.12.0/24 +173.234.53.0/24 +173.236.182.0/24 +173.236.212.0/24 +173.244.209.0/24 +173.244.217.0/24 +173.252.88.0/24 +173.252.105.0/24 +173.252.108.0/24 +173.252.248.0/24 +173.255.209.0/24 +173.255.213.0/24 +174.36.196.0/24 +174.36.228.0/24 +174.37.54.0/24 +174.37.154.0/24 +174.37.175.0/24 +174.37.243.0/24 +178.151.230.0/24 +178.176.156.0/24 +179.60.193.0/24 +180.163.150.0/23 +182.50.139.0/24 +182.79.251.0/24 +184.72.1.0/24 +184.173.136.0/24 +185.45.6.0/23 +185.60.216.0/24 +185.60.218.0/23 +185.158.208.0/24 +186.208.210.0/24 +187.7.116.0/24 +190.5.235.0/24 +192.133.77.0/24 +192.178.24.0/23 +192.178.27.0/24 +192.178.48.0/23 +192.178.50.0/24 +192.178.52.0/24 +192.178.54.0/24 +192.178.56.0/23 +192.178.128.0/24 +192.178.130.0/24 +193.109.164.0/24 +194.78.0.0/24 +196.49.8.0/24 +198.27.124.0/24 +198.44.185.0/24 +199.16.156.0/24 +199.16.158.0/24 +199.59.148.0/23 +199.59.150.0/24 +199.96.58.0/23 +199.96.61.0/24 +199.96.62.0/23 +199.193.116.0/24 +201.0.223.0/24 +202.53.137.0/24 +202.160.128.0/23 +202.160.130.0/24 +202.169.173.0/24 +202.182.98.0/24 +203.66.182.0/24 +203.111.254.0/24 +203.113.51.0/24 +203.113.189.0/24 +203.208.39.0/24 +203.208.40.0/23 +203.208.43.0/24 +203.208.49.0/24 +203.208.50.0/24 +203.233.96.0/24 +204.79.197.0/24 +205.186.152.0/24 +208.31.254.0/24 +208.43.170.0/24 +208.43.237.0/24 +208.77.47.0/24 +208.101.21.0/24 +208.101.60.0/24 +209.85.144.0/22 +209.85.165.0/24 +209.85.200.0/22 +209.85.224.0/24 +209.85.226.0/24 +209.85.232.0/22 +209.95.56.0/24 +210.56.51.0/24 +210.139.253.0/24 +210.209.84.0/24 +211.104.160.0/24 +212.113.52.0/24 +213.59.210.0/24 +216.58.192.0/20 +216.58.208.0/21 +216.58.217.0/24 +216.58.220.0/22 +216.239.32.0/24 +216.239.34.0/24 +216.239.36.0/24 +216.239.38.0/24 +220.181.174.0/24 +""" diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 24a56a0..c891934 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -13,13 +13,15 @@ from cidr4_merger import ( make_groups, make_parent, merge_neighbors, + merge_nodes_cycle, merge_nodes_deprecated, merge_nodes_recursion, - merge_nodes_cycle, reduce_nodes, sort_nodes, ) +from .cidr4_data_for_tests import test_cidr4_data + def test_true(): assert True @@ -378,3 +380,16 @@ def test_merge_nodes_cycle(): ) assert exc_info.type is Cidr4MergerError assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_recursion_vs_cycle(): + required_len = 20 + + test_data = test_cidr4_data.strip().splitlines() + test_nodes = list(map(cidr4_to_node, test_data)) + test_nodes = sort_nodes(test_nodes) + + merged_nodes_recursion = merge_nodes_recursion(test_nodes, required_len) + merged_nodes_cycle = merge_nodes_cycle(test_nodes, required_len) + + assert merged_nodes_recursion == merged_nodes_cycle -- 2.54.0 From 303ddee08f57d2187acca45ac6c130db38e9c8a3 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 19 Jan 2025 01:45:25 +0300 Subject: [PATCH 47/89] remove unnecessary copying of node list --- cidr4_merger.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index d7306c7..7dfa3d0 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -125,23 +125,21 @@ def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: if parent[2] < min_parent[2]: min_single, min_parent = node, parent - new_nodes = [x for x in nodes] - new_nodes.remove(min_single) - new_nodes.append(min_parent) - new_nodes = sort_nodes(new_nodes) - return new_nodes + nodes.remove(min_single) + nodes.append(min_parent) + nodes = sort_nodes(nodes) + return nodes def merge_neighbors( nodes: list[Node], neighbours: list[tuple[Node, Node]] ) -> list[Node]: - new_nodes = [x for x in nodes] for a, b in neighbours: parent = make_parent(a, b) - new_nodes.remove(a) - new_nodes.remove(b) - new_nodes.append(parent) - return sort_nodes(new_nodes) + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + return sort_nodes(nodes) def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: -- 2.54.0 From 2cab2c31a6f40dcc315d19e82463a20934a95b8a Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 17:49:20 +0300 Subject: [PATCH 48/89] add current solution to archive before changes --- archive/_test_cidr4_merger.py | 395 ++++++++++++++++++++++++++++++++++ archive/cidr4_merger.py | 226 +++++++++++++++++++ 2 files changed, 621 insertions(+) create mode 100644 archive/_test_cidr4_merger.py create mode 100644 archive/cidr4_merger.py diff --git a/archive/_test_cidr4_merger.py b/archive/_test_cidr4_merger.py new file mode 100644 index 0000000..c891934 --- /dev/null +++ b/archive/_test_cidr4_merger.py @@ -0,0 +1,395 @@ +import pytest + +from cidr4_merger import ( + Cidr4MergerError, + cidr4_to_node, + find_neighbours_singles, + get_group_with_max_mask_len, + get_net_addr, + get_parent_ip, + have_same_parent, + lift_lonely_node, + make_cidr4, + make_groups, + make_parent, + merge_neighbors, + merge_nodes_cycle, + merge_nodes_deprecated, + merge_nodes_recursion, + reduce_nodes, + sort_nodes, +) + +from .cidr4_data_for_tests import test_cidr4_data + + +def test_true(): + assert True + + +bin_a = "10011000000000001000010000010000" +assert len(bin_a) == 32 +ip_a = int(bin_a, 2) + +bin_b = "10011100000000000000000000101011" +assert len(bin_b) == 32 +ip_b = int(bin_b, 2) + +bin_c = "10011000000000000000000000000000" +assert len(bin_c) == 32 +ip_c = int(bin_c, 2) + +bin_d = "11111100000000000000000000000000" +assert len(bin_c) == 32 +ip_d = int(bin_d, 2) + + +def test_cidr4_to_node(): + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0, 72256000) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0, 0) + + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0, 401219072) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0, 2899902464) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0, 400657408) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0, 520969216) + + assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0, 0) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0, 0) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0, 2147483648) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0, 2147483648) + + +def test_make_cidr4(): + assert make_cidr4(72256256, 24) == "4.78.139.0/24" + assert make_cidr4(0, 32) == "0.0.0.0/32" + + assert make_cidr4(401219072, 24) == "23.234.30.0/24" + assert make_cidr4(2899902464, 19) == "172.217.0.0/19" + assert make_cidr4(400657664, 24) == "23.225.141.0/24" + assert make_cidr4(520969728, 23) == "31.13.94.0/23" + + +def test_get_net_addr(): + assert get_net_addr(ip_a, 5) == ip_c + assert get_net_addr(ip_b, 5) == ip_c + assert get_net_addr(0, 1) == 0 + assert get_net_addr(0, 0) == 0 + + +def test_get_parent_mask(): + assert get_parent_ip(ip_a, 6) == ip_c + assert get_parent_ip(ip_b, 6) == ip_c + assert get_parent_ip(0, 1) == 0 + + with pytest.raises(Exception) as exc_info: + get_parent_ip(0, 0) + assert str(exc_info.value) == "The top of the tree has no parent!" + assert exc_info.type is Cidr4MergerError + + +def test_have_same_parent(): + assert have_same_parent(ip_c, 6, ip_c, 6) is True + assert have_same_parent(ip_c, 6, ip_c, 5) is False + assert have_same_parent(ip_a, 6, ip_d, 6) is False + assert have_same_parent(ip_a, 6, 0, 1) is False + assert have_same_parent(ip_a, 6, 0, 0) is False + + +def test_sort_nodes(): + assert sort_nodes( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (400657664, 24, 0, 400657408), + (520969728, 23, 0, 520969216), + ] + ) == [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + + +def test_get_group_with_max_mask_len(): + assert get_group_with_max_mask_len( + [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + ) == [(400657664, 24, 0, 400657408), (401219072, 24, 0, 401219072)] + + assert get_group_with_max_mask_len( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + ] + ) == [(401219072, 24, 0, 401219072)] + + +def test_make_parent(): + assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) + assert make_parent( + (2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648) + ) == (2147483648, 1, 3, 0) + + with pytest.raises(Exception) as exc_info: + make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) + assert str(exc_info.value) == "Nodes must be neighbors!" + assert exc_info.type is Cidr4MergerError + + +def test_reduce_nodes(): + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + ) == [ + (0, 1, 15, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (2147483648, 1, 0, 0), + ] + ) == [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + + with pytest.raises(Exception) as exc_info: + reduce_nodes( + [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_deprecated(): + assert merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_make_groups(): + nodes = [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + (520963072, 22, 0, 520962048), + (1089054720, 22, 0, 1089054720), + (2899902464, 19, 0, 2899902464), + (2915221504, 19, 0, 2915221504), + ] + assert dict(make_groups(nodes)) == { + 20: [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + ], + 22: [(520963072, 22, 0, 520962048), (1089054720, 22, 0, 1089054720)], + 19: [(2899902464, 19, 0, 2899902464), (2915221504, 19, 0, 2915221504)], + } + + +@pytest.fixture +def nodes_only_neighbours(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + + +@pytest.fixture +def nodes_only_singles(): + return [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def nodes_with_neighbours_n_singles(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def groups_only_neighbours(nodes_only_neighbours): + return make_groups(nodes_only_neighbours) + + +@pytest.fixture +def groups_only_singles(nodes_only_singles): + return make_groups(nodes_only_singles) + + +@pytest.fixture +def groups_with_neighbours_n_singles(nodes_with_neighbours_n_singles): + return make_groups(nodes_with_neighbours_n_singles) + + +def test_find_neighbours_singles( + groups_only_neighbours, + groups_only_singles, + groups_with_neighbours_n_singles, +): + assert find_neighbours_singles(groups_only_neighbours) == ( + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + [], + ) + + assert find_neighbours_singles(groups_only_singles) == ( + [], + [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], + ) + + assert find_neighbours_singles(groups_with_neighbours_n_singles) == ( + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + [(2147483648, 2, 1, 2147483648)], + ) + + +def test_merge_neighbors__only_neighbours(nodes_only_neighbours): + new_nodes = merge_neighbors( + nodes_only_neighbours, + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 1, 3, 0), + ] + + +def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): + new_nodes = merge_neighbors( + nodes_with_neighbours_n_singles, + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 2, 1, 2147483648), + ] + + +def test_lift_lonely_node(nodes_only_singles): + singles = [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)] + new_nodes = lift_lonely_node(nodes_only_singles, singles) + assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] + + +def test_merge_nodes_recursion(): + assert merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_cycle(): + assert merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_recursion_vs_cycle(): + required_len = 20 + + test_data = test_cidr4_data.strip().splitlines() + test_nodes = list(map(cidr4_to_node, test_data)) + test_nodes = sort_nodes(test_nodes) + + merged_nodes_recursion = merge_nodes_recursion(test_nodes, required_len) + merged_nodes_cycle = merge_nodes_cycle(test_nodes, required_len) + + assert merged_nodes_recursion == merged_nodes_cycle diff --git a/archive/cidr4_merger.py b/archive/cidr4_merger.py new file mode 100644 index 0000000..7dfa3d0 --- /dev/null +++ b/archive/cidr4_merger.py @@ -0,0 +1,226 @@ +import cProfile +import sys +from collections import defaultdict + +sys.setrecursionlimit(10_000) + +Node = tuple[int, int, int, int] + + +class Cidr4MergerError(Exception): + pass + + +def get_data(input_file): + with open(input_file, "r") as file: + data = file.read().splitlines() + return data + + +def cidr4_to_node(cidr4: str) -> Node: + ip_address, mask_len = cidr4.strip().split("/") + mask_len = int(mask_len) + a, b, c, d = list(map(int, ip_address.split("."))) + ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 + added_ips_number = 0 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def sort_nodes(nodes: list[Node]) -> list[Node]: + return sorted(nodes, key=lambda x: (x[1], x[0])) + + +def get_net_addr(ip: int, mask_len: int) -> int: + mask = ((1 << mask_len) - 1) << (32 - mask_len) + net_addr = ip & mask + return net_addr + + +def get_parent_ip(ip: int, mask_len: int) -> int: + if mask_len == 0: + raise Cidr4MergerError("The top of the tree has no parent!") + return get_net_addr(ip, mask_len - 1) + + +def have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b) -> bool: + return mask_len_a == mask_len_b and parent_ip_a == parent_ip_b + + +def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: + max_mask_len = max(nodes, key=lambda x: x[1])[1] + return list(filter(lambda x: x[1] == max_mask_len, nodes)) + + +def make_parent(a: Node, b: Node | None = None) -> Node: + ip_a, mask_len_a, added_ips_number_a, parent_ip_a = a + if b: + ip_b, mask_len_b, added_ips_b, parent_ip_b = b + if not have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + raise Cidr4MergerError("Nodes must be neighbors!") + added_ips_number = added_ips_number_a + added_ips_b + else: + added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) + ip = parent_ip_a + mask_len = mask_len_a - 1 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def reduce_nodes(nodes: list[Node]) -> list[Node]: + group = get_group_with_max_mask_len(nodes) + + neighbours = [] + loners = [] + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + loners.append(a) + i += 1 + if i == len(group) - 1: + loners.append(group[i]) + + if neighbours: + zipped = zip(neighbours, map(lambda x: make_parent(x[0], x[1]), neighbours)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + (a, b), parent = min_zipped + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + elif loners: + zipped = zip(loners, map(make_parent, loners)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + a, parent = min_zipped + nodes.remove(a) + nodes.append(parent) + else: + assert False, "Error" + + return sort_nodes(nodes) + + +def merge_nodes_deprecated(nodes: list[Node], required_len: int) -> list[Node]: + while len(nodes) > required_len: + nodes = reduce_nodes(nodes) + return nodes + + +def make_cidr4(ip, mask_len) -> str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}" + + +def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: + # find single whose parent has the least added addresses + min_single, min_parent = singles[0], make_parent(singles[0]) + for node in singles[1:]: + parent = make_parent(node) + if parent[2] < min_parent[2]: + min_single, min_parent = node, parent + + nodes.remove(min_single) + nodes.append(min_parent) + nodes = sort_nodes(nodes) + return nodes + + +def merge_neighbors( + nodes: list[Node], neighbours: list[tuple[Node, Node]] +) -> list[Node]: + for a, b in neighbours: + parent = make_parent(a, b) + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + return sort_nodes(nodes) + + +def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: + neighbours = [] + singles = [] + for group in groups.values(): + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + singles.append(a) + i += 1 + if i == len(group) - 1: + singles.append(group[i]) + return neighbours, singles + + +def make_groups(nodes: list[Node]) -> defaultdict: + groups = defaultdict(list) + for n in nodes: + groups[n[1]].append(n) + return groups + + +def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: + if len(nodes) <= required_len: + return nodes + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + new_nodes = merge_neighbors(nodes, neighbours) + return merge_nodes_recursion(new_nodes, required_len) + new_nodes = lift_lonely_node(nodes, singles) + return merge_nodes_recursion(new_nodes, required_len) + + +def merge_nodes_cycle(nodes_to_merge: list[Node], required_len: int) -> list[Node]: + nodes = [x for x in nodes_to_merge] + while not len(nodes) <= required_len: + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + nodes = merge_neighbors(nodes, neighbours) + elif singles: + nodes = lift_lonely_node(nodes, singles) + else: + raise Cidr4MergerError("Invalid case!") + return nodes + + +def main(): + file = "cidr4.txt" + required_len = 20 + + data = get_data(file) + nodes = list(map(cidr4_to_node, data)) + + nodes = sort_nodes(nodes) + # merged_nodes = merge_nodes_deprecated(nodes, required_len) + # merged_nodes = merge_nodes_recursion(nodes, required_len) + merged_nodes = merge_nodes_cycle(nodes, required_len) + + cidr4s = [] + sum_added_ips = 0 + for ip_value, mask_len, added_ips, _ in merged_nodes: + cidr4s.append(make_cidr4(ip_value, mask_len)) + sum_added_ips += added_ips + + cidr4s_str = "\n".join(cidr4s) + print( + f"Исходный список длины {len(nodes)} сокращен до {len(cidr4s)}\n" + f"Количество добавленных ip адресов: {sum_added_ips:_}\n" + f"Список объединенных cidr4:\n" + f"{cidr4s_str}" + ) + + +if __name__ == "__main__": + cProfile.run("main()") -- 2.54.0 From dfd6e21f2b92d3c55b9bd088f357ce32108b1486 Mon Sep 17 00:00:00 2001 From: Fedor Lyanguzov Date: Wed, 22 Jan 2025 18:21:39 +0300 Subject: [PATCH 49/89] Fast and precise algorithms, with tests --- .activate | 7 ++++ tests/cidr4_merge/test_solutions.py | 56 +++++++++++++++++++++++++++ vpn_manager/cidr4_merge/fast.py | 43 +++++++++++++++++++++ vpn_manager/cidr4_merge/precise.py | 59 +++++++++++++++++++++++++++++ vpn_manager/cidr4_merge/util.py | 20 ++++++++++ 5 files changed, 185 insertions(+) create mode 100644 .activate create mode 100644 tests/cidr4_merge/test_solutions.py create mode 100644 vpn_manager/cidr4_merge/fast.py create mode 100644 vpn_manager/cidr4_merge/precise.py create mode 100644 vpn_manager/cidr4_merge/util.py diff --git a/.activate b/.activate new file mode 100644 index 0000000..a3607ff --- /dev/null +++ b/.activate @@ -0,0 +1,7 @@ + +alias venv='source .venv/Scripts/activate' +alias check='flake8 vpn_manager' +alias format='black vpn_manager' +alias test='pytest' + +venv diff --git a/tests/cidr4_merge/test_solutions.py b/tests/cidr4_merge/test_solutions.py new file mode 100644 index 0000000..091fb9b --- /dev/null +++ b/tests/cidr4_merge/test_solutions.py @@ -0,0 +1,56 @@ +from vpn_manager.cidr4_merge.fast import solution as fast +from vpn_manager.cidr4_merge.precise import solution as precise, f +from vpn_manager.cidr4_merge.util import * + + +def test_true(): + assert True + +#cidrs = list(map(cidr4_to_node, get_data())) + +def test_fast_single_lifting(): + assert ([(0, 30)], 0) ==\ + fast([(0, 31), (2, 31)], 1) + assert ([(0, 29)], 2) ==\ + fast([(0, 30), (4, 31)], 1) + assert ([(0, 29)], 3) ==\ + fast([(0, 30), (4, 32)], 1) + +def test_fast_double_lifting(): + assert ([(0, 29)], 4) ==\ + fast([(0, 31), (4, 31)], 1) + + +def test_fast_subnets(): + assert ([(0, 30)], 0) ==\ + fast([(0, 30), (0, 31)], 1) + assert ([(0, 29)], 0) ==\ + fast([(0, 29), (4, 31)], 1) + + +def test_precise_single_lifting(): + assert ([(0, 30)], 0) ==\ + precise([(0, 31), (2, 31)], 1) + assert ([(0, 29)], 2) ==\ + precise([(0, 30), (4, 31)], 1) + assert ([(0, 29)], 3) ==\ + precise([(0, 30), (4, 32)], 1) + +def test_precise_double_lifting(): + assert ([(0, 29)], 4) ==\ + precise([(0, 31), (4, 31)], 1) + + +def test_precise_subnets(): + assert ([(0, 30)], 0) ==\ + precise([(0, 30), (0, 31)], 1) + assert ([(0, 29)], 0) ==\ + precise([(0, 29), (4, 31)], 1) + +def test_precise_f_single_lifting(): + assert (0, (0, 30, 0)) ==\ + f((0, 31, 0), (2, 31, 0)) + assert (2, (0, 29, 2)) ==\ + f((0, 30, 0), (4, 31, 0)) + assert (3, (0, 29, 3)) ==\ + f((0, 30, 0), (4, 32, 0)) diff --git a/vpn_manager/cidr4_merge/fast.py b/vpn_manager/cidr4_merge/fast.py new file mode 100644 index 0000000..ca65b46 --- /dev/null +++ b/vpn_manager/cidr4_merge/fast.py @@ -0,0 +1,43 @@ +from heapq import heapify, heappush, heappop +from .util import mask, get_data, cidr4_to_node, make_cidr4 + + +def solution(cidrs, M): + h = [] + d = {} + for ip, l in cidrs: + h.append((32-l, ip, l)) + d[(ip, l)] = 0 + heapify(h) + while len(h)>M: + x1, ip1, l1 = heappop(h) + x2, ip2, l2 = h[0] + if l1==l2 and ip1 & mask[l1-1] == ip2 & mask[l2-1]: + heappop(h) + if (ip1 & mask[l1-1], l1-1) not in d: + heappush(h, (x1+1, ip1 & mask[l1-1], l1-1)) + d[(ip1 & mask[l1-1], l1-1)] = d[(ip1, l1)] + d[(ip2, l2)] + del d[(ip2, l2)] + else: + if (ip1 & mask[l1-1], l1-1) not in d: + heappush(h, (x1+1, ip1 & mask[l1-1], l1-1)) + d[(ip1 & mask[l1-1], l1-1)] = d[(ip1, l1)] + 2**x1 + del d[(ip1, l1)] + s = sum(d.values()) + cidrs = list(d.keys()) + return cidrs, s + + +def main(): + M = 20 + a = get_data() + b = list(map(cidr4_to_node, a)) + cidrs, s = solution(b, M) + cidrs = sorted([make_cidr4(*x) for x in cidrs]) + print(cidrs, s, sep='\n') + + +if __name__=='__main__': + import cProfile + main() + diff --git a/vpn_manager/cidr4_merge/precise.py b/vpn_manager/cidr4_merge/precise.py new file mode 100644 index 0000000..8dcae41 --- /dev/null +++ b/vpn_manager/cidr4_merge/precise.py @@ -0,0 +1,59 @@ +from .util import mask, get_data, cidr4_to_node, make_cidr4 + + +def f(x, y): + t = x + b = y + if x[1]>y[1]: + t = y + b = x + ip1, l1, a1 = t + ip2, l2, a2 = b + if ip1 & mask[l1] == ip2 & mask[l1]: + return (0, t) + t1 = t2 = 0 + while not l1==l2: + t2 += 2**(32-l2) + l2 -= 1 + ip2 = ip2 & mask[l2] + while not ip1 & mask[l1-1] == ip2 & mask[l2-1]: + t1 += 2**(32-l1) + l1 -= 1 + ip1 = ip1 & mask[l1] + t2 += 2**(32-l2) + l2 -= 1 + ip2 = ip2 & mask[l2] + r = (ip1 & mask[l1-1], l1-1, a1+a2+t1+t2) + return (t1+t2, r) + + +def solution(cidrs, M): + cidrs = sorted((ip, l, 0) for ip, l in cidrs) + while len(cidrs)>M: + t = (None, float('+inf'), None) + for i, (x, y) in enumerate(zip(cidrs, cidrs[1:])): + m, r = f(x, y) + if m str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}" -- 2.54.0 From 66346431555871bce3851fd2f94c7ceea615bc9d Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 18:23:02 +0300 Subject: [PATCH 50/89] init new algorithm --- cidr4_merger.py | 162 ++---------------- tests/test_cidr4_merger.py | 326 ++----------------------------------- 2 files changed, 31 insertions(+), 457 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 7dfa3d0..fae09ba 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,10 +1,6 @@ import cProfile -import sys -from collections import defaultdict -sys.setrecursionlimit(10_000) - -Node = tuple[int, int, int, int] +Node = tuple[int, int, int] class Cidr4MergerError(Exception): @@ -23,12 +19,11 @@ def cidr4_to_node(cidr4: str) -> Node: a, b, c, d = list(map(int, ip_address.split("."))) ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 added_ips_number = 0 - parent_ip = get_parent_ip(ip, mask_len) - return ip, mask_len, added_ips_number, parent_ip + return ip, mask_len, added_ips_number def sort_nodes(nodes: list[Node]) -> list[Node]: - return sorted(nodes, key=lambda x: (x[1], x[0])) + return sorted(nodes) def get_net_addr(ip: int, mask_len: int) -> int: @@ -43,72 +38,9 @@ def get_parent_ip(ip: int, mask_len: int) -> int: return get_net_addr(ip, mask_len - 1) -def have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b) -> bool: - return mask_len_a == mask_len_b and parent_ip_a == parent_ip_b - - -def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: - max_mask_len = max(nodes, key=lambda x: x[1])[1] - return list(filter(lambda x: x[1] == max_mask_len, nodes)) - - -def make_parent(a: Node, b: Node | None = None) -> Node: - ip_a, mask_len_a, added_ips_number_a, parent_ip_a = a - if b: - ip_b, mask_len_b, added_ips_b, parent_ip_b = b - if not have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): - raise Cidr4MergerError("Nodes must be neighbors!") - added_ips_number = added_ips_number_a + added_ips_b - else: - added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) - ip = parent_ip_a - mask_len = mask_len_a - 1 - parent_ip = get_parent_ip(ip, mask_len) - return ip, mask_len, added_ips_number, parent_ip - - -def reduce_nodes(nodes: list[Node]) -> list[Node]: - group = get_group_with_max_mask_len(nodes) - - neighbours = [] - loners = [] - i = 0 - while i < len(group) - 1: - a, b = group[i], group[i + 1] - ip_a, mask_len_a, _, parent_ip_a = a - ip_b, mask_len_b, _, parent_ip_b = b - if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): - neighbours.append((a, b)) - i += 2 - else: - loners.append(a) - i += 1 - if i == len(group) - 1: - loners.append(group[i]) - - if neighbours: - zipped = zip(neighbours, map(lambda x: make_parent(x[0], x[1]), neighbours)) - min_zipped = min(zipped, key=lambda x: x[1][2]) - (a, b), parent = min_zipped - nodes.remove(a) - nodes.remove(b) - nodes.append(parent) - elif loners: - zipped = zip(loners, map(make_parent, loners)) - min_zipped = min(zipped, key=lambda x: x[1][2]) - a, parent = min_zipped - nodes.remove(a) - nodes.append(parent) - else: - assert False, "Error" - - return sort_nodes(nodes) - - -def merge_nodes_deprecated(nodes: list[Node], required_len: int) -> list[Node]: - while len(nodes) > required_len: - nodes = reduce_nodes(nodes) - return nodes +def make_parent(a: Node, b: Node) -> Node: + ip, mask_len, added_ips_number = None + return ip, mask_len, added_ips_number def make_cidr4(ip, mask_len) -> str: @@ -117,81 +49,14 @@ def make_cidr4(ip, mask_len) -> str: return f"{ip_address}/{mask_len}" -def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: - # find single whose parent has the least added addresses - min_single, min_parent = singles[0], make_parent(singles[0]) - for node in singles[1:]: - parent = make_parent(node) - if parent[2] < min_parent[2]: - min_single, min_parent = node, parent - - nodes.remove(min_single) - nodes.append(min_parent) - nodes = sort_nodes(nodes) - return nodes - - -def merge_neighbors( - nodes: list[Node], neighbours: list[tuple[Node, Node]] -) -> list[Node]: - for a, b in neighbours: - parent = make_parent(a, b) - nodes.remove(a) - nodes.remove(b) - nodes.append(parent) - return sort_nodes(nodes) - - -def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: - neighbours = [] - singles = [] - for group in groups.values(): - i = 0 - while i < len(group) - 1: - a, b = group[i], group[i + 1] - ip_a, mask_len_a, _, parent_ip_a = a - ip_b, mask_len_b, _, parent_ip_b = b - if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): - neighbours.append((a, b)) - i += 2 - else: - singles.append(a) - i += 1 - if i == len(group) - 1: - singles.append(group[i]) - return neighbours, singles - - -def make_groups(nodes: list[Node]) -> defaultdict: - groups = defaultdict(list) - for n in nodes: - groups[n[1]].append(n) - return groups - - -def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: - if len(nodes) <= required_len: - return nodes - groups = make_groups(nodes) - neighbours, singles = find_neighbours_singles(groups) - if neighbours: - new_nodes = merge_neighbors(nodes, neighbours) - return merge_nodes_recursion(new_nodes, required_len) - new_nodes = lift_lonely_node(nodes, singles) - return merge_nodes_recursion(new_nodes, required_len) - - -def merge_nodes_cycle(nodes_to_merge: list[Node], required_len: int) -> list[Node]: +def merge_nodes(nodes_to_merge: list[Node], required_len: int) -> list[Node]: nodes = [x for x in nodes_to_merge] - while not len(nodes) <= required_len: - groups = make_groups(nodes) - neighbours, singles = find_neighbours_singles(groups) - if neighbours: - nodes = merge_neighbors(nodes, neighbours) - elif singles: - nodes = lift_lonely_node(nodes, singles) - else: - raise Cidr4MergerError("Invalid case!") + # преобразовать список нод в список туплов: родитель (ip, mask len, added ips), d_ip = кол-во добавляемых адресов + # найти подходящего родителя (с минимальным значением d_ip), затем мержить два узла: + # 1) если в соседних ветках + # 2) если один из узлов находится в подсети у другого + # повторить пока не достигнем нужного кол-ва узлов + return nodes @@ -203,7 +68,6 @@ def main(): nodes = list(map(cidr4_to_node, data)) nodes = sort_nodes(nodes) - # merged_nodes = merge_nodes_deprecated(nodes, required_len) # merged_nodes = merge_nodes_recursion(nodes, required_len) merged_nodes = merge_nodes_cycle(nodes, required_len) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index c891934..3b1d4d0 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -3,20 +3,10 @@ import pytest from cidr4_merger import ( Cidr4MergerError, cidr4_to_node, - find_neighbours_singles, - get_group_with_max_mask_len, get_net_addr, get_parent_ip, - have_same_parent, - lift_lonely_node, make_cidr4, - make_groups, make_parent, - merge_neighbors, - merge_nodes_cycle, - merge_nodes_deprecated, - merge_nodes_recursion, - reduce_nodes, sort_nodes, ) @@ -45,18 +35,18 @@ ip_d = int(bin_d, 2) def test_cidr4_to_node(): - assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0, 72256000) - assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0, 0) + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) - assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0, 401219072) - assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0, 2899902464) - assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0, 400657408) - assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0, 520969216) + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0) - assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0, 0) - assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0, 0) - assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0, 2147483648) - assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0, 2147483648) + assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0) def test_make_cidr4(): @@ -87,49 +77,22 @@ def test_get_parent_mask(): assert exc_info.type is Cidr4MergerError -def test_have_same_parent(): - assert have_same_parent(ip_c, 6, ip_c, 6) is True - assert have_same_parent(ip_c, 6, ip_c, 5) is False - assert have_same_parent(ip_a, 6, ip_d, 6) is False - assert have_same_parent(ip_a, 6, 0, 1) is False - assert have_same_parent(ip_a, 6, 0, 0) is False - - def test_sort_nodes(): assert sort_nodes( [ - (401219072, 24, 0, 401219072), - (2899902464, 19, 0, 2899902464), - (400657664, 24, 0, 400657408), - (520969728, 23, 0, 520969216), + (401219072, 24, 0), + (2899902464, 19, 0), + (400657664, 24, 0), + (520969728, 23, 0), ] ) == [ - (2899902464, 19, 0, 2899902464), - (520969728, 23, 0, 520969216), - (400657664, 24, 0, 400657408), - (401219072, 24, 0, 401219072), + (2899902464, 19, 0), + (400657664, 24, 0), + (401219072, 24, 0), + (520969728, 23, 0), ] -def test_get_group_with_max_mask_len(): - assert get_group_with_max_mask_len( - [ - (2899902464, 19, 0, 2899902464), - (520969728, 23, 0, 520969216), - (400657664, 24, 0, 400657408), - (401219072, 24, 0, 401219072), - ] - ) == [(400657664, 24, 0, 400657408), (401219072, 24, 0, 401219072)] - - assert get_group_with_max_mask_len( - [ - (401219072, 24, 0, 401219072), - (2899902464, 19, 0, 2899902464), - (520969728, 23, 0, 520969216), - ] - ) == [(401219072, 24, 0, 401219072)] - - def test_make_parent(): assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) assert make_parent( @@ -140,256 +103,3 @@ def test_make_parent(): make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) assert str(exc_info.value) == "Nodes must be neighbors!" assert exc_info.type is Cidr4MergerError - - -def test_reduce_nodes(): - assert reduce_nodes( - [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - ] - ) == [ - (0, 1, 15, 0), - ] - - assert reduce_nodes( - [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ] - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - ] - - assert reduce_nodes( - [ - (0, 2, 12, 0), - (2147483648, 1, 0, 0), - ] - ) == [ - (0, 1, 12 + 2**30, 0), - (2147483648, 1, 0, 0), - ] - - with pytest.raises(Exception) as exc_info: - reduce_nodes( - [ - (0, 1, 12 + 2**30, 0), - (2147483648, 1, 0, 0), - ] - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_merge_nodes_deprecated(): - assert merge_nodes_deprecated( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 2, - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - ] - - with pytest.raises(Exception) as exc_info: - merge_nodes_deprecated( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 1, - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_make_groups(): - nodes = [ - (2398793728, 20, 0, 2398789632), - (2899943424, 20, 0, 2899943424), - (3627728896, 20, 0, 3627728896), - (520963072, 22, 0, 520962048), - (1089054720, 22, 0, 1089054720), - (2899902464, 19, 0, 2899902464), - (2915221504, 19, 0, 2915221504), - ] - assert dict(make_groups(nodes)) == { - 20: [ - (2398793728, 20, 0, 2398789632), - (2899943424, 20, 0, 2899943424), - (3627728896, 20, 0, 3627728896), - ], - 22: [(520963072, 22, 0, 520962048), (1089054720, 22, 0, 1089054720)], - 19: [(2899902464, 19, 0, 2899902464), (2915221504, 19, 0, 2915221504)], - } - - -@pytest.fixture -def nodes_only_neighbours(): - return [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ] - - -@pytest.fixture -def nodes_only_singles(): - return [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - ] - - -@pytest.fixture -def nodes_with_neighbours_n_singles(): - return [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - ] - - -@pytest.fixture -def groups_only_neighbours(nodes_only_neighbours): - return make_groups(nodes_only_neighbours) - - -@pytest.fixture -def groups_only_singles(nodes_only_singles): - return make_groups(nodes_only_singles) - - -@pytest.fixture -def groups_with_neighbours_n_singles(nodes_with_neighbours_n_singles): - return make_groups(nodes_with_neighbours_n_singles) - - -def test_find_neighbours_singles( - groups_only_neighbours, - groups_only_singles, - groups_with_neighbours_n_singles, -): - assert find_neighbours_singles(groups_only_neighbours) == ( - [ - ((0, 2, 12, 0), (1073741824, 2, 3, 0)), - ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), - ], - [], - ) - - assert find_neighbours_singles(groups_only_singles) == ( - [], - [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], - ) - - assert find_neighbours_singles(groups_with_neighbours_n_singles) == ( - [((0, 2, 12, 0), (1073741824, 2, 3, 0))], - [(2147483648, 2, 1, 2147483648)], - ) - - -def test_merge_neighbors__only_neighbours(nodes_only_neighbours): - new_nodes = merge_neighbors( - nodes_only_neighbours, - [ - ((0, 2, 12, 0), (1073741824, 2, 3, 0)), - ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), - ], - ) - assert new_nodes == [ - (0, 1, 15, 0), - (2147483648, 1, 3, 0), - ] - - -def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): - new_nodes = merge_neighbors( - nodes_with_neighbours_n_singles, - [((0, 2, 12, 0), (1073741824, 2, 3, 0))], - ) - assert new_nodes == [ - (0, 1, 15, 0), - (2147483648, 2, 1, 2147483648), - ] - - -def test_lift_lonely_node(nodes_only_singles): - singles = [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)] - new_nodes = lift_lonely_node(nodes_only_singles, singles) - assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] - - -def test_merge_nodes_recursion(): - assert merge_nodes_recursion( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 2, - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - ] - - with pytest.raises(Exception) as exc_info: - merge_nodes_recursion( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 1, - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_merge_nodes_cycle(): - assert merge_nodes_cycle( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 2, - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - ] - - with pytest.raises(Exception) as exc_info: - merge_nodes_cycle( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 1, - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_merge_nodes_recursion_vs_cycle(): - required_len = 20 - - test_data = test_cidr4_data.strip().splitlines() - test_nodes = list(map(cidr4_to_node, test_data)) - test_nodes = sort_nodes(test_nodes) - - merged_nodes_recursion = merge_nodes_recursion(test_nodes, required_len) - merged_nodes_cycle = merge_nodes_cycle(test_nodes, required_len) - - assert merged_nodes_recursion == merged_nodes_cycle -- 2.54.0 From 3ff3843b5856cd093c01c075b8129e7b9dcc7239 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 18:58:18 +0300 Subject: [PATCH 51/89] delete unnecessary test data --- tests/cidr4_data_for_tests.py | 488 ---------------------------------- tests/test_cidr4_merger.py | 2 - 2 files changed, 490 deletions(-) delete mode 100644 tests/cidr4_data_for_tests.py diff --git a/tests/cidr4_data_for_tests.py b/tests/cidr4_data_for_tests.py deleted file mode 100644 index 1e5bf85..0000000 --- a/tests/cidr4_data_for_tests.py +++ /dev/null @@ -1,488 +0,0 @@ -test_cidr4_data = """ -4.78.139.0/24 -23.101.24.0/24 -23.202.231.0/24 -23.217.138.0/24 -23.225.141.0/24 -23.234.30.0/24 -31.13.64.0/24 -31.13.67.0/24 -31.13.68.0/22 -31.13.73.0/24 -31.13.75.0/24 -31.13.76.0/24 -31.13.80.0/21 -31.13.88.0/24 -31.13.90.0/23 -31.13.92.0/24 -31.13.94.0/23 -31.13.96.0/24 -31.13.106.0/24 -31.13.112.0/24 -37.152.2.0/24 -38.121.72.0/24 -39.109.122.0/24 -43.226.16.0/24 -43.245.104.0/24 -45.54.28.0/24 -45.77.186.0/24 -45.114.11.0/24 -45.253.131.0/24 -46.61.154.0/24 -46.134.216.0/24 -47.88.58.0/24 -49.231.55.0/24 -50.23.209.0/24 -50.87.93.0/24 -50.117.117.0/24 -52.58.1.0/24 -52.175.9.0/24 -54.89.135.0/24 -54.144.128.0/24 -54.234.18.0/24 -58.63.233.0/24 -59.18.44.0/24 -59.18.46.0/24 -59.24.3.0/24 -59.188.250.0/24 -61.91.8.0/24 -61.205.119.0/24 -62.0.80.0/24 -64.13.192.0/24 -64.53.242.0/24 -64.233.160.0/21 -64.233.168.0/22 -64.233.176.0/20 -65.49.26.0/24 -65.49.68.0/24 -66.102.1.0/24 -66.220.146.0/23 -66.220.148.0/23 -67.15.100.0/24 -67.15.129.0/24 -67.228.102.0/24 -67.228.235.0/24 -67.230.169.0/24 -69.30.25.0/24 -69.50.221.0/24 -69.63.176.0/24 -69.63.178.0/24 -69.63.180.0/23 -69.63.184.0/24 -69.63.186.0/23 -69.63.190.0/24 -69.162.134.0/24 -69.171.224.0/24 -69.171.227.0/24 -69.171.228.0/23 -69.171.234.0/24 -69.171.242.0/24 -69.171.247.0/24 -69.197.153.0/24 -74.86.3.0/24 -74.86.12.0/24 -74.86.17.0/24 -74.86.118.0/24 -74.86.142.0/24 -74.86.151.0/24 -74.86.226.0/24 -74.86.228.0/24 -74.125.1.0/24 -74.125.2.0/24 -74.125.8.0/24 -74.125.10.0/23 -74.125.13.0/24 -74.125.20.0/23 -74.125.23.0/24 -74.125.24.0/24 -74.125.26.0/24 -74.125.28.0/24 -74.125.31.0/24 -74.125.68.0/22 -74.125.90.0/24 -74.125.96.0/24 -74.125.100.0/24 -74.125.102.0/23 -74.125.104.0/24 -74.125.106.0/24 -74.125.110.0/23 -74.125.124.0/24 -74.125.126.0/23 -74.125.128.0/20 -74.125.153.0/24 -74.125.154.0/23 -74.125.159.0/24 -74.125.164.0/24 -74.125.170.0/24 -74.125.172.0/23 -74.125.192.0/23 -74.125.195.0/24 -74.125.196.0/22 -74.125.200.0/22 -74.125.204.0/23 -74.125.206.0/24 -75.126.33.0/24 -75.126.115.0/24 -75.126.124.0/24 -75.126.135.0/24 -75.126.150.0/24 -75.126.164.0/24 -77.37.252.0/24 -77.120.15.0/24 -80.87.199.0/24 -81.23.20.0/24 -81.23.23.0/24 -81.192.13.0/24 -81.192.191.0/24 -88.191.249.0/24 -90.201.124.0/24 -92.87.232.0/24 -93.179.102.0/24 -94.24.232.0/24 -94.31.189.0/24 -95.59.170.0/24 -96.44.137.0/24 -98.159.108.0/24 -103.39.76.0/24 -103.42.176.0/24 -103.56.16.0/24 -103.73.161.0/24 -103.97.3.0/24 -103.97.176.0/24 -103.200.30.0/23 -103.214.168.0/24 -103.226.246.0/24 -103.228.130.0/24 -103.230.123.0/24 -103.240.180.0/24 -103.240.182.0/24 -103.246.246.0/24 -103.252.114.0/23 -104.16.251.0/24 -104.16.252.0/24 -104.23.124.0/23 -104.31.142.0/24 -104.244.43.0/24 -104.244.45.0/24 -104.244.46.0/24 -107.181.166.0/24 -108.160.161.0/24 -108.160.162.0/23 -108.160.165.0/24 -108.160.166.0/23 -108.160.169.0/24 -108.160.170.0/24 -108.160.172.0/23 -108.177.8.0/21 -108.177.96.0/23 -108.177.98.0/24 -108.177.103.0/24 -108.177.104.0/24 -108.177.111.0/24 -108.177.112.0/24 -108.177.119.0/24 -108.177.120.0/22 -108.177.125.0/24 -108.177.126.0/23 -109.224.41.0/24 -110.164.8.0/24 -111.243.214.0/24 -113.108.239.0/24 -113.171.242.0/24 -114.4.7.0/24 -114.43.24.0/24 -115.126.100.0/24 -116.89.243.0/24 -118.98.30.0/24 -118.98.36.0/24 -118.98.106.0/24 -118.107.180.0/24 -118.184.26.0/24 -118.184.78.0/24 -118.193.202.0/24 -118.193.240.0/24 -119.28.87.0/24 -120.232.233.0/24 -120.232.234.0/24 -120.233.71.0/24 -120.253.250.0/24 -120.253.253.0/24 -120.253.255.0/24 -121.78.42.0/24 -122.10.85.0/24 -122.154.76.0/24 -122.248.226.0/24 -122.252.245.0/24 -124.11.210.0/24 -128.121.146.0/24 -128.121.243.0/24 -128.242.240.0/24 -128.242.245.0/24 -128.242.250.0/24 -130.211.15.0/24 -142.250.0.0/23 -142.250.4.0/24 -142.250.8.0/22 -142.250.12.0/23 -142.250.27.0/24 -142.250.28.0/24 -142.250.30.0/23 -142.250.64.0/20 -142.250.80.0/23 -142.250.96.0/21 -142.250.105.0/24 -142.250.107.0/24 -142.250.110.0/23 -142.250.112.0/22 -142.250.123.0/24 -142.250.125.0/24 -142.250.126.0/24 -142.250.128.0/24 -142.250.136.0/24 -142.250.138.0/24 -142.250.141.0/24 -142.250.142.0/24 -142.250.145.0/24 -142.250.147.0/24 -142.250.148.0/23 -142.250.150.0/24 -142.250.152.0/23 -142.250.157.0/24 -142.250.158.0/23 -142.250.176.0/20 -142.250.192.0/21 -142.250.200.0/23 -142.250.203.0/24 -142.250.204.0/22 -142.250.217.0/24 -142.250.218.0/23 -142.251.0.0/23 -142.251.2.0/24 -142.251.4.0/23 -142.251.6.0/24 -142.251.8.0/23 -142.251.10.0/24 -142.251.12.0/24 -142.251.15.0/24 -142.251.16.0/24 -142.251.18.0/24 -142.251.31.0/24 -142.251.32.0/22 -142.251.36.0/23 -142.251.39.0/24 -142.251.40.0/22 -142.251.45.0/24 -142.251.46.0/23 -142.251.107.0/24 -142.251.111.0/24 -142.251.112.0/24 -142.251.116.0/23 -142.251.120.0/24 -142.251.128.0/23 -142.251.130.0/24 -142.251.132.0/22 -142.251.140.0/22 -142.251.161.0/24 -142.251.162.0/23 -142.251.164.0/22 -142.251.168.0/24 -142.251.170.0/23 -142.251.172.0/22 -142.251.176.0/22 -142.251.180.0/24 -142.251.182.0/23 -142.251.184.0/22 -142.251.188.0/24 -142.251.208.0/23 -142.251.211.0/24 -142.251.214.0/23 -142.251.216.0/24 -142.251.218.0/24 -142.251.220.0/22 -145.255.14.0/24 -148.163.48.0/24 -150.107.3.0/24 -154.0.29.0/24 -154.83.14.0/23 -154.85.102.0/24 -154.92.16.0/24 -156.233.67.0/24 -157.240.0.0/22 -157.240.6.0/23 -157.240.8.0/22 -157.240.12.0/23 -157.240.15.0/24 -157.240.16.0/23 -157.240.18.0/24 -157.240.20.0/23 -159.65.107.0/24 -159.106.121.0/24 -159.138.20.0/24 -162.125.1.0/24 -162.125.2.0/24 -162.125.6.0/23 -162.125.8.0/24 -162.125.17.0/24 -162.125.18.0/24 -162.125.32.0/24 -162.125.34.0/24 -162.125.80.0/24 -162.125.82.0/23 -162.220.12.0/24 -168.143.162.0/24 -168.143.171.0/24 -172.217.0.0/19 -172.217.129.0/24 -172.217.130.0/24 -172.217.133.0/24 -172.217.135.0/24 -172.217.160.0/20 -172.217.192.0/22 -172.217.197.0/24 -172.217.203.0/24 -172.217.204.0/24 -172.217.212.0/24 -172.217.214.0/23 -172.217.218.0/23 -172.217.222.0/24 -172.253.58.0/24 -172.253.62.0/23 -172.253.112.0/21 -172.253.120.0/24 -172.253.122.0/23 -172.253.124.0/22 -173.194.4.0/23 -173.194.12.0/24 -173.194.22.0/24 -173.194.28.0/23 -173.194.31.0/24 -173.194.49.0/24 -173.194.51.0/24 -173.194.54.0/23 -173.194.59.0/24 -173.194.65.0/24 -173.194.66.0/23 -173.194.68.0/23 -173.194.70.0/24 -173.194.73.0/24 -173.194.74.0/24 -173.194.76.0/22 -173.194.135.0/24 -173.194.150.0/24 -173.194.154.0/24 -173.194.161.0/24 -173.194.162.0/23 -173.194.164.0/24 -173.194.166.0/23 -173.194.174.0/23 -173.194.178.0/24 -173.194.182.0/23 -173.194.184.0/23 -173.194.187.0/24 -173.194.188.0/24 -173.194.190.0/23 -173.194.192.0/19 -173.208.182.0/24 -173.231.12.0/24 -173.234.53.0/24 -173.236.182.0/24 -173.236.212.0/24 -173.244.209.0/24 -173.244.217.0/24 -173.252.88.0/24 -173.252.105.0/24 -173.252.108.0/24 -173.252.248.0/24 -173.255.209.0/24 -173.255.213.0/24 -174.36.196.0/24 -174.36.228.0/24 -174.37.54.0/24 -174.37.154.0/24 -174.37.175.0/24 -174.37.243.0/24 -178.151.230.0/24 -178.176.156.0/24 -179.60.193.0/24 -180.163.150.0/23 -182.50.139.0/24 -182.79.251.0/24 -184.72.1.0/24 -184.173.136.0/24 -185.45.6.0/23 -185.60.216.0/24 -185.60.218.0/23 -185.158.208.0/24 -186.208.210.0/24 -187.7.116.0/24 -190.5.235.0/24 -192.133.77.0/24 -192.178.24.0/23 -192.178.27.0/24 -192.178.48.0/23 -192.178.50.0/24 -192.178.52.0/24 -192.178.54.0/24 -192.178.56.0/23 -192.178.128.0/24 -192.178.130.0/24 -193.109.164.0/24 -194.78.0.0/24 -196.49.8.0/24 -198.27.124.0/24 -198.44.185.0/24 -199.16.156.0/24 -199.16.158.0/24 -199.59.148.0/23 -199.59.150.0/24 -199.96.58.0/23 -199.96.61.0/24 -199.96.62.0/23 -199.193.116.0/24 -201.0.223.0/24 -202.53.137.0/24 -202.160.128.0/23 -202.160.130.0/24 -202.169.173.0/24 -202.182.98.0/24 -203.66.182.0/24 -203.111.254.0/24 -203.113.51.0/24 -203.113.189.0/24 -203.208.39.0/24 -203.208.40.0/23 -203.208.43.0/24 -203.208.49.0/24 -203.208.50.0/24 -203.233.96.0/24 -204.79.197.0/24 -205.186.152.0/24 -208.31.254.0/24 -208.43.170.0/24 -208.43.237.0/24 -208.77.47.0/24 -208.101.21.0/24 -208.101.60.0/24 -209.85.144.0/22 -209.85.165.0/24 -209.85.200.0/22 -209.85.224.0/24 -209.85.226.0/24 -209.85.232.0/22 -209.95.56.0/24 -210.56.51.0/24 -210.139.253.0/24 -210.209.84.0/24 -211.104.160.0/24 -212.113.52.0/24 -213.59.210.0/24 -216.58.192.0/20 -216.58.208.0/21 -216.58.217.0/24 -216.58.220.0/22 -216.239.32.0/24 -216.239.34.0/24 -216.239.36.0/24 -216.239.38.0/24 -220.181.174.0/24 -""" diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index 3b1d4d0..3147ffa 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -10,8 +10,6 @@ from cidr4_merger import ( sort_nodes, ) -from .cidr4_data_for_tests import test_cidr4_data - def test_true(): assert True -- 2.54.0 From c659e96d3f4a833ad362f661fc7e7ce14c250d0b Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 21:47:31 +0300 Subject: [PATCH 52/89] add cidr4_merger algorithm to the others --- README.md | 9 ++++++++- tests/{ => cidr4_merge}/test_cidr4_merger.py | 5 +++-- .../cidr4_merge/cidr4_merger.py | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) rename tests/{ => cidr4_merge}/test_cidr4_merger.py (97%) rename cidr4_merger.py => vpn_manager/cidr4_merge/cidr4_merger.py (95%) diff --git a/README.md b/README.md index c5f97b1..77379f2 100644 --- a/README.md +++ b/README.md @@ -26,4 +26,11 @@ run # script declared in pyproject.toml $ example run -``` \ No newline at end of file +``` + +Run different merging algorithms: +``` +python -m vpn_manager.cidr4_merge.cidr4_merger +python -m vpn_manager.cidr4_merge.fast +python -m vpn_manager.cidr4_merge.precise +``` diff --git a/tests/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py similarity index 97% rename from tests/test_cidr4_merger.py rename to tests/cidr4_merge/test_cidr4_merger.py index 3147ffa..85490d4 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -1,6 +1,6 @@ import pytest -from cidr4_merger import ( +from vpn_manager.cidr4_merge.cidr4_merger import ( Cidr4MergerError, cidr4_to_node, get_net_addr, @@ -84,13 +84,14 @@ def test_sort_nodes(): (520969728, 23, 0), ] ) == [ - (2899902464, 19, 0), (400657664, 24, 0), (401219072, 24, 0), (520969728, 23, 0), + (2899902464, 19, 0), ] +@pytest.mark.skip(reason="broken") def test_make_parent(): assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) assert make_parent( diff --git a/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py similarity index 95% rename from cidr4_merger.py rename to vpn_manager/cidr4_merge/cidr4_merger.py index fae09ba..db4163c 100644 --- a/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -68,8 +68,7 @@ def main(): nodes = list(map(cidr4_to_node, data)) nodes = sort_nodes(nodes) - # merged_nodes = merge_nodes_recursion(nodes, required_len) - merged_nodes = merge_nodes_cycle(nodes, required_len) + merged_nodes = merge_nodes(nodes, required_len) cidr4s = [] sum_added_ips = 0 -- 2.54.0 From 05c78ad399e7aa1262ced4fc9c9839b54bcfc6ed Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 21:56:19 +0300 Subject: [PATCH 53/89] delete added_ips_number info from node --- tests/cidr4_merge/test_cidr4_merger.py | 20 ++++++++++---------- vpn_manager/cidr4_merge/cidr4_merger.py | 11 +++++------ 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 85490d4..6800752 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -33,18 +33,18 @@ ip_d = int(bin_d, 2) def test_cidr4_to_node(): - assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) - assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24) + assert cidr4_to_node("0.0.0.0/32") == (0, 32) - assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0) - assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0) - assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0) - assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0) + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23) - assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0) - assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0) - assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0) - assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0) + assert cidr4_to_node("0.0.0.0/2") == (0, 2) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2) def test_make_cidr4(): diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index db4163c..77f7212 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -1,6 +1,6 @@ import cProfile -Node = tuple[int, int, int] +Node = tuple[int, int] class Cidr4MergerError(Exception): @@ -15,11 +15,10 @@ def get_data(input_file): def cidr4_to_node(cidr4: str) -> Node: ip_address, mask_len = cidr4.strip().split("/") - mask_len = int(mask_len) a, b, c, d = list(map(int, ip_address.split("."))) ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 - added_ips_number = 0 - return ip, mask_len, added_ips_number + mask_len = int(mask_len) + return ip, mask_len def sort_nodes(nodes: list[Node]) -> list[Node]: @@ -39,8 +38,8 @@ def get_parent_ip(ip: int, mask_len: int) -> int: def make_parent(a: Node, b: Node) -> Node: - ip, mask_len, added_ips_number = None - return ip, mask_len, added_ips_number + ip, mask_len = 0, 0 + return ip, mask_len def make_cidr4(ip, mask_len) -> str: -- 2.54.0 From d591470a0ddf42bd0ac5d8de77b5908e7835562d Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 22:04:47 +0300 Subject: [PATCH 54/89] correct the algorithm plan --- vpn_manager/cidr4_merge/cidr4_merger.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 77f7212..4104d62 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -50,10 +50,12 @@ def make_cidr4(ip, mask_len) -> str: def merge_nodes(nodes_to_merge: list[Node], required_len: int) -> list[Node]: nodes = [x for x in nodes_to_merge] - # преобразовать список нод в список туплов: родитель (ip, mask len, added ips), d_ip = кол-во добавляемых адресов - # найти подходящего родителя (с минимальным значением d_ip), затем мержить два узла: - # 1) если в соседних ветках - # 2) если один из узлов находится в подсети у другого + # Преобразовать список нод в список туплов: родитель (ip, mask len), кол-во добавляемых адресов d_ip + # найти подходящего родителя (с минимальным значением d_ip) и индекс + # затем мержить два узла: + # - т.е. удалить элемент с индексом i + # - еще раз удалить элемент с индексом i + # - добавить родителя перед элементом с индексом i # повторить пока не достигнем нужного кол-ва узлов return nodes -- 2.54.0 From faa2849e6ba4ee5770d1819c684a975bbd379264 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 22:25:09 +0300 Subject: [PATCH 55/89] fix sort nodes test --- tests/cidr4_merge/test_cidr4_merger.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 6800752..7796379 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -78,16 +78,16 @@ def test_get_parent_mask(): def test_sort_nodes(): assert sort_nodes( [ - (401219072, 24, 0), - (2899902464, 19, 0), - (400657664, 24, 0), - (520969728, 23, 0), + (401219072, 24), + (2899902464, 19), + (400657664, 24), + (520969728, 23), ] ) == [ - (400657664, 24, 0), - (401219072, 24, 0), - (520969728, 23, 0), - (2899902464, 19, 0), + (400657664, 24), + (401219072, 24), + (520969728, 23), + (2899902464, 19), ] -- 2.54.0 From 725b80544aaa95c330ea73540b057fcfc3cf1b82 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 23:12:21 +0300 Subject: [PATCH 56/89] add find_parent function --- tests/cidr4_merge/test_cidr4_merger.py | 74 ++++++++++--------------- vpn_manager/cidr4_merge/cidr4_merger.py | 28 +++++----- 2 files changed, 42 insertions(+), 60 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 7796379..2e917c9 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -3,10 +3,8 @@ import pytest from vpn_manager.cidr4_merge.cidr4_merger import ( Cidr4MergerError, cidr4_to_node, - get_net_addr, - get_parent_ip, + find_parent, make_cidr4, - make_parent, sort_nodes, ) @@ -15,23 +13,6 @@ def test_true(): assert True -bin_a = "10011000000000001000010000010000" -assert len(bin_a) == 32 -ip_a = int(bin_a, 2) - -bin_b = "10011100000000000000000000101011" -assert len(bin_b) == 32 -ip_b = int(bin_b, 2) - -bin_c = "10011000000000000000000000000000" -assert len(bin_c) == 32 -ip_c = int(bin_c, 2) - -bin_d = "11111100000000000000000000000000" -assert len(bin_c) == 32 -ip_d = int(bin_d, 2) - - def test_cidr4_to_node(): assert cidr4_to_node("4.78.139.0/24") == (72256256, 24) assert cidr4_to_node("0.0.0.0/32") == (0, 32) @@ -57,24 +38,6 @@ def test_make_cidr4(): assert make_cidr4(520969728, 23) == "31.13.94.0/23" -def test_get_net_addr(): - assert get_net_addr(ip_a, 5) == ip_c - assert get_net_addr(ip_b, 5) == ip_c - assert get_net_addr(0, 1) == 0 - assert get_net_addr(0, 0) == 0 - - -def test_get_parent_mask(): - assert get_parent_ip(ip_a, 6) == ip_c - assert get_parent_ip(ip_b, 6) == ip_c - assert get_parent_ip(0, 1) == 0 - - with pytest.raises(Exception) as exc_info: - get_parent_ip(0, 0) - assert str(exc_info.value) == "The top of the tree has no parent!" - assert exc_info.type is Cidr4MergerError - - def test_sort_nodes(): assert sort_nodes( [ @@ -91,14 +54,33 @@ def test_sort_nodes(): ] -@pytest.mark.skip(reason="broken") -def test_make_parent(): - assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) - assert make_parent( - (2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648) - ) == (2147483648, 1, 3, 0) +def test_find_parent(): + assert find_parent((0, 2), (1073741824, 2)) == (0, 1) + assert find_parent((2147483648, 2), (3221225472, 2)) == (2147483648, 1) + assert find_parent((0, 2), (3221225472, 2)) == (0, 0) + assert find_parent((1, 32), (6, 32)) == (0, 29) + + +def test_find_parent__with_exception(): + with pytest.raises(Exception) as exc_info: + find_parent((0, 32), (0, 29)) + assert ( + str(exc_info.value) + == "Error! Trying to find common parent of network and subnet! parent_node=(0, 29), a=(0, 32), b=(0, 29)." + ) + assert exc_info.type is Cidr4MergerError + with pytest.raises(Exception) as exc_info: + find_parent((0, 1), (1073741824, 2)) + assert ( + str(exc_info.value) + == "Error! Trying to find common parent of network and subnet! parent_node=(0, 1), a=(0, 1), b=(1073741824, 2)." + ) + assert exc_info.type is Cidr4MergerError with pytest.raises(Exception) as exc_info: - make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) - assert str(exc_info.value) == "Nodes must be neighbors!" + find_parent((0, 0), (3221225472, 2)) + assert ( + str(exc_info.value) + == "Error! Trying to find common parent of network and subnet! parent_node=(0, 0), a=(0, 0), b=(3221225472, 2)." + ) assert exc_info.type is Cidr4MergerError diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 4104d62..142d39b 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -25,21 +25,21 @@ def sort_nodes(nodes: list[Node]) -> list[Node]: return sorted(nodes) -def get_net_addr(ip: int, mask_len: int) -> int: +def find_parent(a: Node, b: Node) -> Node: + ip_a, mask_len_a = a + ip_b, mask_len_b = b + mask_len = min(mask_len_a, mask_len_b) mask = ((1 << mask_len) - 1) << (32 - mask_len) - net_addr = ip & mask - return net_addr - - -def get_parent_ip(ip: int, mask_len: int) -> int: - if mask_len == 0: - raise Cidr4MergerError("The top of the tree has no parent!") - return get_net_addr(ip, mask_len - 1) - - -def make_parent(a: Node, b: Node) -> Node: - ip, mask_len = 0, 0 - return ip, mask_len + while ip_a & mask != ip_b & mask: + mask_len -= 1 + mask = ((1 << mask_len) - 1) << (32 - mask_len) + ip = ip_a & mask + parent_node = ip, mask_len + if parent_node == a or parent_node == b: + raise Cidr4MergerError( + f"Error! Trying to find common parent of network and subnet! {parent_node=}, {a=}, {b=}." + ) + return parent_node def make_cidr4(ip, mask_len) -> str: -- 2.54.0 From 5a8f98041f780c1a9a09a02ea08d3b56a0dc99f3 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 23:27:47 +0300 Subject: [PATCH 57/89] correct mask recalculation when reducing its length --- vpn_manager/cidr4_merge/cidr4_merger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 142d39b..6da4b2d 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -32,7 +32,7 @@ def find_parent(a: Node, b: Node) -> Node: mask = ((1 << mask_len) - 1) << (32 - mask_len) while ip_a & mask != ip_b & mask: mask_len -= 1 - mask = ((1 << mask_len) - 1) << (32 - mask_len) + mask = (mask << 1) & ((1 << 32) - 1) ip = ip_a & mask parent_node = ip, mask_len if parent_node == a or parent_node == b: -- 2.54.0 From 81f741ee4ed251c65080cd88b0e04ac9ce2c9027 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 23 Jan 2025 09:42:30 +0300 Subject: [PATCH 58/89] fix plan --- vpn_manager/cidr4_merge/cidr4_merger.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 6da4b2d..57a5b4d 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -48,8 +48,11 @@ def make_cidr4(ip, mask_len) -> str: return f"{ip_address}/{mask_len}" -def merge_nodes(nodes_to_merge: list[Node], required_len: int) -> list[Node]: +def merge_nodes( + nodes_to_merge: list[Node], required_len: int +) -> tuple[list[Node], int]: nodes = [x for x in nodes_to_merge] + sum_dip = 0 # Преобразовать список нод в список туплов: родитель (ip, mask len), кол-во добавляемых адресов d_ip # найти подходящего родителя (с минимальным значением d_ip) и индекс # затем мержить два узла: @@ -58,7 +61,7 @@ def merge_nodes(nodes_to_merge: list[Node], required_len: int) -> list[Node]: # - добавить родителя перед элементом с индексом i # повторить пока не достигнем нужного кол-ва узлов - return nodes + return nodes, sum_dip def main(): @@ -69,18 +72,14 @@ def main(): nodes = list(map(cidr4_to_node, data)) nodes = sort_nodes(nodes) - merged_nodes = merge_nodes(nodes, required_len) + merged_nodes, sum_dip = merge_nodes(nodes, required_len) - cidr4s = [] - sum_added_ips = 0 - for ip_value, mask_len, added_ips, _ in merged_nodes: - cidr4s.append(make_cidr4(ip_value, mask_len)) - sum_added_ips += added_ips + cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] cidr4s_str = "\n".join(cidr4s) print( f"Исходный список длины {len(nodes)} сокращен до {len(cidr4s)}\n" - f"Количество добавленных ip адресов: {sum_added_ips:_}\n" + f"Количество добавленных ip адресов: {sum_dip:_}\n" f"Список объединенных cidr4:\n" f"{cidr4s_str}" ) -- 2.54.0 From 474b6779d81c191d00cdb99b7fb5814c604fbc55 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 23 Jan 2025 10:15:30 +0300 Subject: [PATCH 59/89] add calc_dip function --- tests/cidr4_merge/test_cidr4_merger.py | 7 +++++++ vpn_manager/cidr4_merge/cidr4_merger.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 2e917c9..28acd42 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -2,6 +2,7 @@ import pytest from vpn_manager.cidr4_merge.cidr4_merger import ( Cidr4MergerError, + calc_dip, cidr4_to_node, find_parent, make_cidr4, @@ -84,3 +85,9 @@ def test_find_parent__with_exception(): == "Error! Trying to find common parent of network and subnet! parent_node=(0, 0), a=(0, 0), b=(3221225472, 2)." ) assert exc_info.type is Cidr4MergerError + + +def test_calc_dip(): + assert calc_dip(26, 27) == 67108864 + assert calc_dip(26, 28) == 201326592 + assert calc_dip(26, 29) == 469762048 diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 57a5b4d..f92b9e2 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -42,6 +42,14 @@ def find_parent(a: Node, b: Node) -> Node: return parent_node +def calc_dip(mask_len_a: int, mask_len_b: int) -> int: + dip = 0 + while mask_len_a < mask_len_b: + dip += 1 << mask_len_a + mask_len_a += 1 + return dip + + def make_cidr4(ip, mask_len) -> str: lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] ip_address = ".".join(lst) -- 2.54.0 From adc569af213fa6a95ff4ad6efcd1b66bd3980c0d Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 23 Jan 2025 10:20:36 +0300 Subject: [PATCH 60/89] fix calc_dip --- tests/cidr4_merge/test_cidr4_merger.py | 7 +++++++ vpn_manager/cidr4_merge/cidr4_merger.py | 7 ++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 28acd42..319976b 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -88,6 +88,13 @@ def test_find_parent__with_exception(): def test_calc_dip(): + assert calc_dip(26, 26) == 0 + assert calc_dip(26, 27) == 67108864 + assert calc_dip(27, 26) == 67108864 + assert calc_dip(26, 28) == 201326592 + assert calc_dip(28, 26) == 201326592 + assert calc_dip(26, 29) == 469762048 + assert calc_dip(29, 26) == 469762048 diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index f92b9e2..a2d7802 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -44,9 +44,10 @@ def find_parent(a: Node, b: Node) -> Node: def calc_dip(mask_len_a: int, mask_len_b: int) -> int: dip = 0 - while mask_len_a < mask_len_b: - dip += 1 << mask_len_a - mask_len_a += 1 + len_min, len_max = sorted((mask_len_a, mask_len_b)) + while len_min < len_max: + dip += 1 << len_min + len_min += 1 return dip -- 2.54.0 From 91d3556eee8887bcbf8bf7b259402400838a4505 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Thu, 23 Jan 2025 18:34:13 +0300 Subject: [PATCH 61/89] fix calc_dip, add merge_two_nodes --- tests/cidr4_merge/test_cidr4_merger.py | 22 +++++++++++++++------- vpn_manager/cidr4_merge/cidr4_merger.py | 21 ++++++++++++++++----- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 319976b..32add5f 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -6,6 +6,7 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( cidr4_to_node, find_parent, make_cidr4, + merge_two_nodes, sort_nodes, ) @@ -88,13 +89,20 @@ def test_find_parent__with_exception(): def test_calc_dip(): + assert calc_dip(32, 32) == 0 + assert calc_dip(32, 31) == 2 + assert calc_dip(32, 30) == 4 + assert calc_dip(31, 30) == 2 + assert calc_dip(30, 29) == 4 + assert calc_dip(29, 31) == 6 assert calc_dip(26, 26) == 0 + assert calc_dip(3, 2) == 2**30 - 2**29 + assert calc_dip(3, 1) == 2**31 - 2**29 - assert calc_dip(26, 27) == 67108864 - assert calc_dip(27, 26) == 67108864 - assert calc_dip(26, 28) == 201326592 - assert calc_dip(28, 26) == 201326592 - - assert calc_dip(26, 29) == 469762048 - assert calc_dip(29, 26) == 469762048 +def test_merge_two_nodes(): + assert merge_two_nodes((0, 32), (1, 32)) == ((0, 31), 0) + assert merge_two_nodes((0, 32), (2, 32)) == ((0, 30), 4) + assert merge_two_nodes((0, 32), (5, 32)) == ((0, 29), 8) + assert merge_two_nodes((3, 32), (4, 32)) == ((0, 29), 8) + assert merge_two_nodes((0, 32), (4, 30)) == ((0, 29), 4) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index a2d7802..c3bfeab 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -43,14 +43,25 @@ def find_parent(a: Node, b: Node) -> Node: def calc_dip(mask_len_a: int, mask_len_b: int) -> int: - dip = 0 - len_min, len_max = sorted((mask_len_a, mask_len_b)) - while len_min < len_max: - dip += 1 << len_min - len_min += 1 + # a = 2 ** (32 - mask_len_a) if mask_len_a != 32 else 0 + # b = 2 ** (32 - mask_len_b) if mask_len_b != 32 else 0 + a = 1 << (32 - mask_len_a) if mask_len_a != 32 else 0 + b = 1 << (32 - mask_len_b) if mask_len_b != 32 else 0 + dip = abs(a - b) return dip +def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: + ip_a, mask_len_a = node_a + ip_b, mask_len_b = node_b + parent_node = find_parent(node_a, node_b) + _, mask_len_p = parent_node + dip_a = calc_dip(mask_len_a, mask_len_p + 1) + dip_b = calc_dip(mask_len_b, mask_len_p + 1) + dip = dip_a + dip_b + return parent_node, dip + + def make_cidr4(ip, mask_len) -> str: lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] ip_address = ".".join(lst) -- 2.54.0 From d987b39bbd876e2d1ade2bb91e1a9ce90273704c Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 10:05:11 +0300 Subject: [PATCH 62/89] add merge_nodes function --- tests/cidr4_merge/test_cidr4_merger.py | 61 +++++++++++++++++++------ vpn_manager/cidr4_merge/cidr4_merger.py | 48 ++++++++++--------- 2 files changed, 75 insertions(+), 34 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 32add5f..2c87518 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -6,6 +6,7 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( cidr4_to_node, find_parent, make_cidr4, + merge_nodes, merge_two_nodes, sort_nodes, ) @@ -89,20 +90,54 @@ def test_find_parent__with_exception(): def test_calc_dip(): - assert calc_dip(32, 32) == 0 - assert calc_dip(32, 31) == 2 - assert calc_dip(32, 30) == 4 - assert calc_dip(31, 30) == 2 - assert calc_dip(30, 29) == 4 - assert calc_dip(29, 31) == 6 - assert calc_dip(26, 26) == 0 - assert calc_dip(3, 2) == 2**30 - 2**29 - assert calc_dip(3, 1) == 2**31 - 2**29 + assert calc_dip(32, 32, 31) == 0 + assert calc_dip(32, 32, 30) == 2 + assert calc_dip(32, 32, 29) == 6 + assert calc_dip(32, 30, 29) == 3 + assert calc_dip(32, 31, 29) == 5 + assert calc_dip(32, 31, 29) == 5 + + assert calc_dip(2, 2, 1) == 0 + assert calc_dip(2, 2, 0) == 2**31 + assert calc_dip(3, 3, 1) == 2**30 def test_merge_two_nodes(): assert merge_two_nodes((0, 32), (1, 32)) == ((0, 31), 0) - assert merge_two_nodes((0, 32), (2, 32)) == ((0, 30), 4) - assert merge_two_nodes((0, 32), (5, 32)) == ((0, 29), 8) - assert merge_two_nodes((3, 32), (4, 32)) == ((0, 29), 8) - assert merge_two_nodes((0, 32), (4, 30)) == ((0, 29), 4) + assert merge_two_nodes((0, 32), (2, 32)) == ((0, 30), 2) + assert merge_two_nodes((0, 32), (5, 32)) == ((0, 29), 6) + assert merge_two_nodes((3, 32), (4, 32)) == ((0, 29), 6) + assert merge_two_nodes((0, 32), (4, 30)) == ((0, 29), 3) + assert merge_two_nodes((0, 32), (6, 31)) == ((0, 29), 5) + + +def test_merge_nodes(): + assert merge_nodes( + [ + (0, 32), + (3, 32), + (4, 32), + ], + 2, + ) == ([(0, 30), (4, 32)], 2) + + # т.е. сейчас алгоритм работает так, что он может оставить соседей + assert merge_nodes( + [ + (0, 32), + (3, 32), + (4, 32), + (7, 32), + ], + 2, + ) == ([(0, 30), (4, 30)], 4) + + assert merge_nodes( + [ + (0, 32), + (3, 32), + (4, 32), + (7, 32), + ], + 1, + ) == ([(0, 29)], 4) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index c3bfeab..61d5893 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -42,13 +42,17 @@ def find_parent(a: Node, b: Node) -> Node: return parent_node -def calc_dip(mask_len_a: int, mask_len_b: int) -> int: - # a = 2 ** (32 - mask_len_a) if mask_len_a != 32 else 0 - # b = 2 ** (32 - mask_len_b) if mask_len_b != 32 else 0 - a = 1 << (32 - mask_len_a) if mask_len_a != 32 else 0 - b = 1 << (32 - mask_len_b) if mask_len_b != 32 else 0 - dip = abs(a - b) - return dip +def calc_dip(mask_len_a: int, mask_len_b: int, mask_len_p: int) -> int: + mask_len = mask_len_p + 1 + dip_a = 0 + while mask_len_a > mask_len: + dip_a += 2 ** (32 - mask_len_a) + mask_len_a -= 1 + dip_b = 0 + while mask_len_b > mask_len: + dip_b += 2 ** (32 - mask_len_b) + mask_len_b -= 1 + return dip_a + dip_b def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: @@ -56,9 +60,7 @@ def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: ip_b, mask_len_b = node_b parent_node = find_parent(node_a, node_b) _, mask_len_p = parent_node - dip_a = calc_dip(mask_len_a, mask_len_p + 1) - dip_b = calc_dip(mask_len_b, mask_len_p + 1) - dip = dip_a + dip_b + dip = calc_dip(mask_len_a, mask_len_b, mask_len_p) return parent_node, dip @@ -68,18 +70,22 @@ def make_cidr4(ip, mask_len) -> str: return f"{ip_address}/{mask_len}" -def merge_nodes( - nodes_to_merge: list[Node], required_len: int -) -> tuple[list[Node], int]: - nodes = [x for x in nodes_to_merge] +def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: sum_dip = 0 - # Преобразовать список нод в список туплов: родитель (ip, mask len), кол-во добавляемых адресов d_ip - # найти подходящего родителя (с минимальным значением d_ip) и индекс - # затем мержить два узла: - # - т.е. удалить элемент с индексом i - # - еще раз удалить элемент с индексом i - # - добавить родителя перед элементом с индексом i - # повторить пока не достигнем нужного кол-ва узлов + while len(nodes) > required_len: + min_tuple = None, (None, float("inf")) + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if dip < min_tuple[1][1]: + min_tuple = i, (parent_node, dip) + idx, (parent_node, dip) = min_tuple + nodes = nodes[:idx] + [parent_node] + nodes[idx + 2 :] + sum_dip += dip + + # for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + # parent_node, dip = merge_two_nodes(a, b) + # assert parent_node != a + # assert parent_node != b return nodes, sum_dip -- 2.54.0 From f40d47989a1586f88392013e9abf893c0691153f Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 11:31:12 +0300 Subject: [PATCH 63/89] make similar output --- vpn_manager/cidr4_merge/cidr4_merger.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 61d5893..9a80ccb 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -101,14 +101,7 @@ def main(): merged_nodes, sum_dip = merge_nodes(nodes, required_len) cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] - - cidr4s_str = "\n".join(cidr4s) - print( - f"Исходный список длины {len(nodes)} сокращен до {len(cidr4s)}\n" - f"Количество добавленных ip адресов: {sum_dip:_}\n" - f"Список объединенных cidr4:\n" - f"{cidr4s_str}" - ) + print(sorted(cidr4s), sum_dip, sep="\n") if __name__ == "__main__": -- 2.54.0 From 55c71c6eb21c21a84af3f8aa8eb8a7a57d6844c3 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 11:35:03 +0300 Subject: [PATCH 64/89] use functions from util --- vpn_manager/cidr4_merge/cidr4_merger.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 9a80ccb..286438c 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -1,5 +1,7 @@ import cProfile +from .util import cidr4_to_node, get_data, make_cidr4 + Node = tuple[int, int] @@ -7,20 +9,6 @@ class Cidr4MergerError(Exception): pass -def get_data(input_file): - with open(input_file, "r") as file: - data = file.read().splitlines() - return data - - -def cidr4_to_node(cidr4: str) -> Node: - ip_address, mask_len = cidr4.strip().split("/") - a, b, c, d = list(map(int, ip_address.split("."))) - ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 - mask_len = int(mask_len) - return ip, mask_len - - def sort_nodes(nodes: list[Node]) -> list[Node]: return sorted(nodes) @@ -64,12 +52,6 @@ def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: return parent_node, dip -def make_cidr4(ip, mask_len) -> str: - lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] - ip_address = ".".join(lst) - return f"{ip_address}/{mask_len}" - - def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: sum_dip = 0 while len(nodes) > required_len: -- 2.54.0 From 7c8ecea9800ab5b027cad3eaf85101d1b6311359 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 11:36:34 +0300 Subject: [PATCH 65/89] delete sort nodes function --- vpn_manager/cidr4_merge/cidr4_merger.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 286438c..cc9528d 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -9,10 +9,6 @@ class Cidr4MergerError(Exception): pass -def sort_nodes(nodes: list[Node]) -> list[Node]: - return sorted(nodes) - - def find_parent(a: Node, b: Node) -> Node: ip_a, mask_len_a = a ip_b, mask_len_b = b @@ -79,7 +75,7 @@ def main(): data = get_data(file) nodes = list(map(cidr4_to_node, data)) - nodes = sort_nodes(nodes) + nodes = sorted(nodes) merged_nodes, sum_dip = merge_nodes(nodes, required_len) cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] -- 2.54.0 From a160ffd3a21546b1a938ff7d297a18baf88440a4 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 11:38:17 +0300 Subject: [PATCH 66/89] add cosmetic changes --- vpn_manager/cidr4_merge/cidr4_merger.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index cc9528d..c98700e 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -69,15 +69,10 @@ def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: def main(): - file = "cidr4.txt" required_len = 20 - - data = get_data(file) - nodes = list(map(cidr4_to_node, data)) - - nodes = sorted(nodes) + data = get_data() + nodes = sorted(map(cidr4_to_node, data)) merged_nodes, sum_dip = merge_nodes(nodes, required_len) - cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] print(sorted(cidr4s), sum_dip, sep="\n") -- 2.54.0 From 68327db96c5cc098f7c8b3ae61c98d2245d34f96 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 11:59:35 +0300 Subject: [PATCH 67/89] fix tests (delete sort_nodes from tests) --- tests/cidr4_merge/test_cidr4_merger.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 2c87518..10c372e 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -8,7 +8,6 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( make_cidr4, merge_nodes, merge_two_nodes, - sort_nodes, ) @@ -41,22 +40,6 @@ def test_make_cidr4(): assert make_cidr4(520969728, 23) == "31.13.94.0/23" -def test_sort_nodes(): - assert sort_nodes( - [ - (401219072, 24), - (2899902464, 19), - (400657664, 24), - (520969728, 23), - ] - ) == [ - (400657664, 24), - (401219072, 24), - (520969728, 23), - (2899902464, 19), - ] - - def test_find_parent(): assert find_parent((0, 2), (1073741824, 2)) == (0, 1) assert find_parent((2147483648, 2), (3221225472, 2)) == (2147483648, 1) -- 2.54.0 From 6c43344893e3990a76097207afe47e2294c141f0 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 12:13:55 +0300 Subject: [PATCH 68/89] make calc_dip faster --- vpn_manager/cidr4_merge/cidr4_merger.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index c98700e..655d6c1 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -27,16 +27,14 @@ def find_parent(a: Node, b: Node) -> Node: def calc_dip(mask_len_a: int, mask_len_b: int, mask_len_p: int) -> int: - mask_len = mask_len_p + 1 - dip_a = 0 - while mask_len_a > mask_len: - dip_a += 2 ** (32 - mask_len_a) - mask_len_a -= 1 - dip_b = 0 - while mask_len_b > mask_len: - dip_b += 2 ** (32 - mask_len_b) - mask_len_b -= 1 - return dip_a + dip_b + def dip(mla, mlp): + m = mlp + 1 + res = 1 << (mla - m) + res -= 1 + res <<= 32 - mla + return res + + return dip(mask_len_a, mask_len_p) + dip(mask_len_b, mask_len_p) def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: -- 2.54.0 From 75d400f52ec7563e870831ac7f437611860390b2 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 13:30:37 +0300 Subject: [PATCH 69/89] add find subnets example --- archive/find_subnets_example.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 archive/find_subnets_example.md diff --git a/archive/find_subnets_example.md b/archive/find_subnets_example.md new file mode 100644 index 0000000..4c2940b --- /dev/null +++ b/archive/find_subnets_example.md @@ -0,0 +1,14 @@ +```python +def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: + subnets = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if parent_node == a or parent_node == b: + subnets.append((a, b)) + return subnets + + +def ensure_no_subnets(nodes: list[Node]): + if subnets := find_subnets(nodes): + raise Cidr4MergerError(f"There are subnets! {subnets=}") +``` -- 2.54.0 From 811432a2f57bf675178675c7a8b2f1a059b9ea1f Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 13:34:47 +0300 Subject: [PATCH 70/89] add cosmetic changes in merge_two_nodes --- vpn_manager/cidr4_merge/cidr4_merger.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 655d6c1..e250f17 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -38,11 +38,8 @@ def calc_dip(mask_len_a: int, mask_len_b: int, mask_len_p: int) -> int: def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: - ip_a, mask_len_a = node_a - ip_b, mask_len_b = node_b parent_node = find_parent(node_a, node_b) - _, mask_len_p = parent_node - dip = calc_dip(mask_len_a, mask_len_b, mask_len_p) + dip = calc_dip(node_a[1], node_b[1], parent_node[1]) return parent_node, dip -- 2.54.0 From f9f711ca0bbbe517f987d06591820f28324fe61e Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 13:49:00 +0300 Subject: [PATCH 71/89] add insurance functions example --- archive/find_subnets_example.md | 14 ------------ archive/insurance_functions_example.md | 30 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 14 deletions(-) delete mode 100644 archive/find_subnets_example.md create mode 100644 archive/insurance_functions_example.md diff --git a/archive/find_subnets_example.md b/archive/find_subnets_example.md deleted file mode 100644 index 4c2940b..0000000 --- a/archive/find_subnets_example.md +++ /dev/null @@ -1,14 +0,0 @@ -```python -def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: - subnets = [] - for i, (a, b) in enumerate(zip(nodes, nodes[1:])): - parent_node, dip = merge_two_nodes(a, b) - if parent_node == a or parent_node == b: - subnets.append((a, b)) - return subnets - - -def ensure_no_subnets(nodes: list[Node]): - if subnets := find_subnets(nodes): - raise Cidr4MergerError(f"There are subnets! {subnets=}") -``` diff --git a/archive/insurance_functions_example.md b/archive/insurance_functions_example.md new file mode 100644 index 0000000..8b1a256 --- /dev/null +++ b/archive/insurance_functions_example.md @@ -0,0 +1,30 @@ +```python +def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: + subnets = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if parent_node == a or parent_node == b: + subnets.append((a, b)) + return subnets + + +def ensure_no_subnets(nodes: list[Node]): + if subnets := find_subnets(nodes): + raise Cidr4MergerError(f"There are subnets! {subnets=}") +``` + + +```python +def find_neighbors(nodes: list[Node]) -> list[tuple[Node, Node]]: + neighbors = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if parent_node[1] + 1 == a[1] == b[1]: + neighbors.append((a, b)) + return neighbors + + +def ensure_no_neighbors(nodes: list[Node]): + if neighbors := find_neighbors(nodes): + raise Cidr4MergerError(f"There are neighbors! {neighbors=}") +``` -- 2.54.0 From 735a3c02168b8cee9cd881cc83e6f20f5cb401b5 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Fri, 24 Jan 2025 14:01:47 +0300 Subject: [PATCH 72/89] take into account the case when neighbors could remain --- tests/cidr4_merge/test_cidr4_merger.py | 29 +++++++++++++++++++++++-- vpn_manager/cidr4_merge/cidr4_merger.py | 20 ++++++++++++----- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 10c372e..f4a9652 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -4,6 +4,7 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( Cidr4MergerError, calc_dip, cidr4_to_node, + find_neighbors, find_parent, make_cidr4, merge_nodes, @@ -94,6 +95,31 @@ def test_merge_two_nodes(): assert merge_two_nodes((0, 32), (6, 31)) == ((0, 29), 5) +def test_find_neighbors(): + assert find_neighbors([(0, 32), (2, 32), (4, 32), (6, 32)]) == [] + assert find_neighbors([(0, 32), (1, 32)]) == [(0, (0, 32), (1, 32))] + assert find_neighbors( + [ + (0, 32), + (1, 32), + (6, 32), + (7, 32), + ] + ) == [ + (0, (0, 32), (1, 32)), + (2, (6, 32), (7, 32)), + ] + assert find_neighbors( + [ + (0, 32), + (1, 32), + (6, 32), + ] + ) == [ + (0, (0, 32), (1, 32)), + ] + + def test_merge_nodes(): assert merge_nodes( [ @@ -104,7 +130,6 @@ def test_merge_nodes(): 2, ) == ([(0, 30), (4, 32)], 2) - # т.е. сейчас алгоритм работает так, что он может оставить соседей assert merge_nodes( [ (0, 32), @@ -113,7 +138,7 @@ def test_merge_nodes(): (7, 32), ], 2, - ) == ([(0, 30), (4, 30)], 4) + ) == ([(0, 29)], 4) assert merge_nodes( [ diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index e250f17..2a5ede9 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -55,21 +55,31 @@ def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: nodes = nodes[:idx] + [parent_node] + nodes[idx + 2 :] sum_dip += dip - # for i, (a, b) in enumerate(zip(nodes, nodes[1:])): - # parent_node, dip = merge_two_nodes(a, b) - # assert parent_node != a - # assert parent_node != b + while neighbors := find_neighbors(nodes): + idx, a, b = neighbors[0] + parent_node, dip = merge_two_nodes(a, b) + nodes = nodes[:idx] + [parent_node] + nodes[idx + 2 :] + sum_dip += dip return nodes, sum_dip +def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: + neighbors = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if parent_node[1] + 1 == a[1] == b[1]: + neighbors.append((i, a, b)) + return neighbors + + def main(): required_len = 20 data = get_data() nodes = sorted(map(cidr4_to_node, data)) merged_nodes, sum_dip = merge_nodes(nodes, required_len) cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] - print(sorted(cidr4s), sum_dip, sep="\n") + print(cidr4s, sum_dip, sep="\n") if __name__ == "__main__": -- 2.54.0 From eccc03f48da8802d240e45e24a43ec4281e79abf Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 25 Jan 2025 10:15:42 +0300 Subject: [PATCH 73/89] add cosmetic changes (more comfortable work with arguments) --- vpn_manager/cidr4_merge/cidr4_merger.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 2a5ede9..6b3a5e0 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -46,12 +46,12 @@ def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: sum_dip = 0 while len(nodes) > required_len: - min_tuple = None, (None, float("inf")) + min_tuple = None, None, float("inf") for i, (a, b) in enumerate(zip(nodes, nodes[1:])): parent_node, dip = merge_two_nodes(a, b) - if dip < min_tuple[1][1]: - min_tuple = i, (parent_node, dip) - idx, (parent_node, dip) = min_tuple + if dip < min_tuple[2]: + min_tuple = i, parent_node, dip + idx, parent_node, dip = min_tuple nodes = nodes[:idx] + [parent_node] + nodes[idx + 2 :] sum_dip += dip -- 2.54.0 From bc8ddf1be492746b35dbd83248255af9f06f783f Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sat, 25 Jan 2025 10:28:03 +0300 Subject: [PATCH 74/89] add insurance functions, delete raise exception from find_parent function --- tests/cidr4_merge/test_cidr4_merger.py | 30 +++---------------------- vpn_manager/cidr4_merge/cidr4_merger.py | 25 +++++++++++++++++---- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index f4a9652..49df2f3 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -1,7 +1,4 @@ -import pytest - from vpn_manager.cidr4_merge.cidr4_merger import ( - Cidr4MergerError, calc_dip, cidr4_to_node, find_neighbors, @@ -47,30 +44,9 @@ def test_find_parent(): assert find_parent((0, 2), (3221225472, 2)) == (0, 0) assert find_parent((1, 32), (6, 32)) == (0, 29) - -def test_find_parent__with_exception(): - with pytest.raises(Exception) as exc_info: - find_parent((0, 32), (0, 29)) - assert ( - str(exc_info.value) - == "Error! Trying to find common parent of network and subnet! parent_node=(0, 29), a=(0, 32), b=(0, 29)." - ) - assert exc_info.type is Cidr4MergerError - with pytest.raises(Exception) as exc_info: - find_parent((0, 1), (1073741824, 2)) - assert ( - str(exc_info.value) - == "Error! Trying to find common parent of network and subnet! parent_node=(0, 1), a=(0, 1), b=(1073741824, 2)." - ) - assert exc_info.type is Cidr4MergerError - - with pytest.raises(Exception) as exc_info: - find_parent((0, 0), (3221225472, 2)) - assert ( - str(exc_info.value) - == "Error! Trying to find common parent of network and subnet! parent_node=(0, 0), a=(0, 0), b=(3221225472, 2)." - ) - assert exc_info.type is Cidr4MergerError + assert find_parent((0, 32), (0, 29)) == (0, 29) + assert find_parent((0, 1), (1073741824, 2)) == (0, 1) + assert find_parent((0, 0), (3221225472, 2)) == (0, 0) def test_calc_dip(): diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 6b3a5e0..b7cbe74 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -19,10 +19,6 @@ def find_parent(a: Node, b: Node) -> Node: mask = (mask << 1) & ((1 << 32) - 1) ip = ip_a & mask parent_node = ip, mask_len - if parent_node == a or parent_node == b: - raise Cidr4MergerError( - f"Error! Trying to find common parent of network and subnet! {parent_node=}, {a=}, {b=}." - ) return parent_node @@ -64,6 +60,20 @@ def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: return nodes, sum_dip +def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: + subnets = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if parent_node == a or parent_node == b: + subnets.append((a, b)) + return subnets + + +def ensure_no_subnets(nodes: list[Node]): + if subnets := find_subnets(nodes): + raise Cidr4MergerError(f"There are subnets! {subnets=}") + + def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: neighbors = [] for i, (a, b) in enumerate(zip(nodes, nodes[1:])): @@ -73,10 +83,17 @@ def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: return neighbors +def ensure_no_neighbors(nodes: list[Node]): + if neighbors := find_neighbors(nodes): + raise Cidr4MergerError(f"There are neighbors! {neighbors=}") + + def main(): required_len = 20 data = get_data() nodes = sorted(map(cidr4_to_node, data)) + ensure_no_subnets(nodes) + ensure_no_neighbors(nodes) merged_nodes, sum_dip = merge_nodes(nodes, required_len) cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] print(cidr4s, sum_dip, sep="\n") -- 2.54.0 From fa9b4073b7719bf23b5fbbf82e8fb0683af92d1f Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:04:37 +0300 Subject: [PATCH 75/89] move how to launch algorithms info to the folder with algorithms --- README.md | 7 ------- vpn_manager/cidr4_merge/README.md | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 vpn_manager/cidr4_merge/README.md diff --git a/README.md b/README.md index 77379f2..9e19b31 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,3 @@ run $ example run ``` - -Run different merging algorithms: -``` -python -m vpn_manager.cidr4_merge.cidr4_merger -python -m vpn_manager.cidr4_merge.fast -python -m vpn_manager.cidr4_merge.precise -``` diff --git a/vpn_manager/cidr4_merge/README.md b/vpn_manager/cidr4_merge/README.md new file mode 100644 index 0000000..b3c65b0 --- /dev/null +++ b/vpn_manager/cidr4_merge/README.md @@ -0,0 +1,6 @@ +Run merging algorithms: +``` +python -m vpn_manager.cidr4_merge.cidr4_merger +python -m vpn_manager.cidr4_merge.fast +python -m vpn_manager.cidr4_merge.precise +``` -- 2.54.0 From 731d77f0e3b811b45f877099d7c0c3df4f61eb7a Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:24:34 +0300 Subject: [PATCH 76/89] shorten variable names in calc_dip function --- vpn_manager/cidr4_merge/cidr4_merger.py | 28 ++++++++++++------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index b7cbe74..3ba34b8 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -10,27 +10,25 @@ class Cidr4MergerError(Exception): def find_parent(a: Node, b: Node) -> Node: - ip_a, mask_len_a = a - ip_b, mask_len_b = b - mask_len = min(mask_len_a, mask_len_b) - mask = ((1 << mask_len) - 1) << (32 - mask_len) - while ip_a & mask != ip_b & mask: - mask_len -= 1 + ia, la = a + ib, lb = b + min_l = min(la, lb) + mask = ((1 << min_l) - 1) << (32 - min_l) + while ia & mask != ib & mask: + min_l -= 1 mask = (mask << 1) & ((1 << 32) - 1) - ip = ip_a & mask - parent_node = ip, mask_len - return parent_node + return ia & mask, min_l -def calc_dip(mask_len_a: int, mask_len_b: int, mask_len_p: int) -> int: - def dip(mla, mlp): - m = mlp + 1 - res = 1 << (mla - m) +def calc_dip(la: int, lb: int, lp: int) -> int: + def dip(l1, lp): + m = lp + 1 + res = 1 << (l1 - m) res -= 1 - res <<= 32 - mla + res <<= 32 - l1 return res - return dip(mask_len_a, mask_len_p) + dip(mask_len_b, mask_len_p) + return dip(la, lp) + dip(lb, lp) def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: -- 2.54.0 From fee38af6909c79d7bcfc6f54883e5f3b8fb58503 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:27:55 +0300 Subject: [PATCH 77/89] shorten variable names in merge_two_nodes function --- vpn_manager/cidr4_merge/cidr4_merger.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 3ba34b8..bd976f1 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -31,10 +31,10 @@ def calc_dip(la: int, lb: int, lp: int) -> int: return dip(la, lp) + dip(lb, lp) -def merge_two_nodes(node_a: Node, node_b: Node) -> tuple[Node, int]: - parent_node = find_parent(node_a, node_b) - dip = calc_dip(node_a[1], node_b[1], parent_node[1]) - return parent_node, dip +def merge_two_nodes(a: Node, b: Node) -> tuple[Node, int]: + p = find_parent(a, b) + dip = calc_dip(a[1], b[1], p[1]) + return p, dip def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: -- 2.54.0 From 04b4d14a30ad23a9b2d6e06d6e6dca40b460805a Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:30:58 +0300 Subject: [PATCH 78/89] shorten variable names in merge_nodes function --- vpn_manager/cidr4_merge/cidr4_merger.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index bd976f1..74524d4 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -37,22 +37,22 @@ def merge_two_nodes(a: Node, b: Node) -> tuple[Node, int]: return p, dip -def merge_nodes(nodes: list[Node], required_len: int) -> tuple[list[Node], int]: +def merge_nodes(nodes: list[Node], req_len: int) -> tuple[list[Node], int]: sum_dip = 0 - while len(nodes) > required_len: - min_tuple = None, None, float("inf") + while len(nodes) > req_len: + min_t = None, None, float("inf") for i, (a, b) in enumerate(zip(nodes, nodes[1:])): - parent_node, dip = merge_two_nodes(a, b) - if dip < min_tuple[2]: - min_tuple = i, parent_node, dip - idx, parent_node, dip = min_tuple - nodes = nodes[:idx] + [parent_node] + nodes[idx + 2 :] + p, dip = merge_two_nodes(a, b) + if dip < min_t[2]: + min_t = i, p, dip + i, p, dip = min_t + nodes = nodes[:i] + [p] + nodes[i + 2 :] sum_dip += dip - while neighbors := find_neighbors(nodes): - idx, a, b = neighbors[0] - parent_node, dip = merge_two_nodes(a, b) - nodes = nodes[:idx] + [parent_node] + nodes[idx + 2 :] + while nbs := find_neighbors(nodes): + i, a, b = nbs[0] + p, dip = merge_two_nodes(a, b) + nodes = nodes[:i] + [p] + nodes[i + 2 :] sum_dip += dip return nodes, sum_dip -- 2.54.0 From 52c5fb679e3a3b129a125bb3e21b0bc0e74ee4cb Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:39:31 +0300 Subject: [PATCH 79/89] rename merge_nodes function to solution --- tests/cidr4_merge/test_cidr4_merger.py | 8 ++++---- vpn_manager/cidr4_merge/cidr4_merger.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 49df2f3..23995c4 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -4,8 +4,8 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( find_neighbors, find_parent, make_cidr4, - merge_nodes, merge_two_nodes, + solution, ) @@ -97,7 +97,7 @@ def test_find_neighbors(): def test_merge_nodes(): - assert merge_nodes( + assert solution( [ (0, 32), (3, 32), @@ -106,7 +106,7 @@ def test_merge_nodes(): 2, ) == ([(0, 30), (4, 32)], 2) - assert merge_nodes( + assert solution( [ (0, 32), (3, 32), @@ -116,7 +116,7 @@ def test_merge_nodes(): 2, ) == ([(0, 29)], 4) - assert merge_nodes( + assert solution( [ (0, 32), (3, 32), diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 74524d4..024df9b 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -37,7 +37,7 @@ def merge_two_nodes(a: Node, b: Node) -> tuple[Node, int]: return p, dip -def merge_nodes(nodes: list[Node], req_len: int) -> tuple[list[Node], int]: +def solution(nodes: list[Node], req_len: int) -> tuple[list[Node], int]: sum_dip = 0 while len(nodes) > req_len: min_t = None, None, float("inf") @@ -92,7 +92,7 @@ def main(): nodes = sorted(map(cidr4_to_node, data)) ensure_no_subnets(nodes) ensure_no_neighbors(nodes) - merged_nodes, sum_dip = merge_nodes(nodes, required_len) + merged_nodes, sum_dip = solution(nodes, required_len) cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] print(cidr4s, sum_dip, sep="\n") -- 2.54.0 From e0ecafa9d0ba168ec33d9e933cc42a6d72f795f3 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:45:05 +0300 Subject: [PATCH 80/89] changed the order of functions --- tests/cidr4_merge/test_cidr4_merger.py | 18 +++---- vpn_manager/cidr4_merge/cidr4_merger.py | 62 ++++++++++++------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 23995c4..502c567 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -4,7 +4,7 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( find_neighbors, find_parent, make_cidr4, - merge_two_nodes, + merge_nodes, solution, ) @@ -62,13 +62,13 @@ def test_calc_dip(): assert calc_dip(3, 3, 1) == 2**30 -def test_merge_two_nodes(): - assert merge_two_nodes((0, 32), (1, 32)) == ((0, 31), 0) - assert merge_two_nodes((0, 32), (2, 32)) == ((0, 30), 2) - assert merge_two_nodes((0, 32), (5, 32)) == ((0, 29), 6) - assert merge_two_nodes((3, 32), (4, 32)) == ((0, 29), 6) - assert merge_two_nodes((0, 32), (4, 30)) == ((0, 29), 3) - assert merge_two_nodes((0, 32), (6, 31)) == ((0, 29), 5) +def test_merge_nodes(): + assert merge_nodes((0, 32), (1, 32)) == ((0, 31), 0) + assert merge_nodes((0, 32), (2, 32)) == ((0, 30), 2) + assert merge_nodes((0, 32), (5, 32)) == ((0, 29), 6) + assert merge_nodes((3, 32), (4, 32)) == ((0, 29), 6) + assert merge_nodes((0, 32), (4, 30)) == ((0, 29), 3) + assert merge_nodes((0, 32), (6, 31)) == ((0, 29), 5) def test_find_neighbors(): @@ -96,7 +96,7 @@ def test_find_neighbors(): ] -def test_merge_nodes(): +def test_solution(): assert solution( [ (0, 32), diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 024df9b..baf5457 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -9,6 +9,34 @@ class Cidr4MergerError(Exception): pass +def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: + subnets = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_nodes(a, b) + if parent_node == a or parent_node == b: + subnets.append((a, b)) + return subnets + + +def ensure_no_subnets(nodes: list[Node]): + if subnets := find_subnets(nodes): + raise Cidr4MergerError(f"There are subnets! {subnets=}") + + +def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: + neighbors = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_nodes(a, b) + if parent_node[1] + 1 == a[1] == b[1]: + neighbors.append((i, a, b)) + return neighbors + + +def ensure_no_neighbors(nodes: list[Node]): + if neighbors := find_neighbors(nodes): + raise Cidr4MergerError(f"There are neighbors! {neighbors=}") + + def find_parent(a: Node, b: Node) -> Node: ia, la = a ib, lb = b @@ -31,7 +59,7 @@ def calc_dip(la: int, lb: int, lp: int) -> int: return dip(la, lp) + dip(lb, lp) -def merge_two_nodes(a: Node, b: Node) -> tuple[Node, int]: +def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: p = find_parent(a, b) dip = calc_dip(a[1], b[1], p[1]) return p, dip @@ -42,7 +70,7 @@ def solution(nodes: list[Node], req_len: int) -> tuple[list[Node], int]: while len(nodes) > req_len: min_t = None, None, float("inf") for i, (a, b) in enumerate(zip(nodes, nodes[1:])): - p, dip = merge_two_nodes(a, b) + p, dip = merge_nodes(a, b) if dip < min_t[2]: min_t = i, p, dip i, p, dip = min_t @@ -51,41 +79,13 @@ def solution(nodes: list[Node], req_len: int) -> tuple[list[Node], int]: while nbs := find_neighbors(nodes): i, a, b = nbs[0] - p, dip = merge_two_nodes(a, b) + p, dip = merge_nodes(a, b) nodes = nodes[:i] + [p] + nodes[i + 2 :] sum_dip += dip return nodes, sum_dip -def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: - subnets = [] - for i, (a, b) in enumerate(zip(nodes, nodes[1:])): - parent_node, dip = merge_two_nodes(a, b) - if parent_node == a or parent_node == b: - subnets.append((a, b)) - return subnets - - -def ensure_no_subnets(nodes: list[Node]): - if subnets := find_subnets(nodes): - raise Cidr4MergerError(f"There are subnets! {subnets=}") - - -def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: - neighbors = [] - for i, (a, b) in enumerate(zip(nodes, nodes[1:])): - parent_node, dip = merge_two_nodes(a, b) - if parent_node[1] + 1 == a[1] == b[1]: - neighbors.append((i, a, b)) - return neighbors - - -def ensure_no_neighbors(nodes: list[Node]): - if neighbors := find_neighbors(nodes): - raise Cidr4MergerError(f"There are neighbors! {neighbors=}") - - def main(): required_len = 20 data = get_data() -- 2.54.0 From a4991518ba5574fa3d7f026f595e61067188f079 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:48:08 +0300 Subject: [PATCH 81/89] move internal functions to the function where they are used --- vpn_manager/cidr4_merge/cidr4_merger.py | 42 ++++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index baf5457..a443322 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -37,29 +37,27 @@ def ensure_no_neighbors(nodes: list[Node]): raise Cidr4MergerError(f"There are neighbors! {neighbors=}") -def find_parent(a: Node, b: Node) -> Node: - ia, la = a - ib, lb = b - min_l = min(la, lb) - mask = ((1 << min_l) - 1) << (32 - min_l) - while ia & mask != ib & mask: - min_l -= 1 - mask = (mask << 1) & ((1 << 32) - 1) - return ia & mask, min_l - - -def calc_dip(la: int, lb: int, lp: int) -> int: - def dip(l1, lp): - m = lp + 1 - res = 1 << (l1 - m) - res -= 1 - res <<= 32 - l1 - return res - - return dip(la, lp) + dip(lb, lp) - - def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: + def find_parent(a: Node, b: Node) -> Node: + ia, la = a + ib, lb = b + min_l = min(la, lb) + mask = ((1 << min_l) - 1) << (32 - min_l) + while ia & mask != ib & mask: + min_l -= 1 + mask = (mask << 1) & ((1 << 32) - 1) + return ia & mask, min_l + + def calc_dip(la: int, lb: int, lp: int) -> int: + def dip(l1, lp): + m = lp + 1 + res = 1 << (l1 - m) + res -= 1 + res <<= 32 - l1 + return res + + return dip(la, lp) + dip(lb, lp) + p = find_parent(a, b) dip = calc_dip(a[1], b[1], p[1]) return p, dip -- 2.54.0 From 0b56caffac0bdc6548215bf2b59d5c39d2e72d96 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 10:52:45 +0300 Subject: [PATCH 82/89] fix tests after removing functions --- tests/cidr4_merge/test_cidr4_merger.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 502c567..6a50aab 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -1,8 +1,6 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( - calc_dip, cidr4_to_node, find_neighbors, - find_parent, make_cidr4, merge_nodes, solution, @@ -38,30 +36,6 @@ def test_make_cidr4(): assert make_cidr4(520969728, 23) == "31.13.94.0/23" -def test_find_parent(): - assert find_parent((0, 2), (1073741824, 2)) == (0, 1) - assert find_parent((2147483648, 2), (3221225472, 2)) == (2147483648, 1) - assert find_parent((0, 2), (3221225472, 2)) == (0, 0) - assert find_parent((1, 32), (6, 32)) == (0, 29) - - assert find_parent((0, 32), (0, 29)) == (0, 29) - assert find_parent((0, 1), (1073741824, 2)) == (0, 1) - assert find_parent((0, 0), (3221225472, 2)) == (0, 0) - - -def test_calc_dip(): - assert calc_dip(32, 32, 31) == 0 - assert calc_dip(32, 32, 30) == 2 - assert calc_dip(32, 32, 29) == 6 - assert calc_dip(32, 30, 29) == 3 - assert calc_dip(32, 31, 29) == 5 - assert calc_dip(32, 31, 29) == 5 - - assert calc_dip(2, 2, 1) == 0 - assert calc_dip(2, 2, 0) == 2**31 - assert calc_dip(3, 3, 1) == 2**30 - - def test_merge_nodes(): assert merge_nodes((0, 32), (1, 32)) == ((0, 31), 0) assert merge_nodes((0, 32), (2, 32)) == ((0, 30), 2) -- 2.54.0 From 6f745c2d6129fc946f5662bf6266234d29045c52 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 11:04:23 +0300 Subject: [PATCH 83/89] remove unnecessary annotations --- vpn_manager/cidr4_merge/cidr4_merger.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index a443322..07abd8f 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -38,7 +38,7 @@ def ensure_no_neighbors(nodes: list[Node]): def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: - def find_parent(a: Node, b: Node) -> Node: + def find_parent(a, b) -> Node: ia, la = a ib, lb = b min_l = min(la, lb) @@ -48,7 +48,7 @@ def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: mask = (mask << 1) & ((1 << 32) - 1) return ia & mask, min_l - def calc_dip(la: int, lb: int, lp: int) -> int: + def calc_dip(la, lb, lp): def dip(l1, lp): m = lp + 1 res = 1 << (l1 - m) -- 2.54.0 From 3e6c8525c40f49bc959b25b4fb053f959082fba8 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 11:05:19 +0300 Subject: [PATCH 84/89] remove unnecessary annotations --- vpn_manager/cidr4_merge/cidr4_merger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 07abd8f..5c7fc02 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -38,7 +38,7 @@ def ensure_no_neighbors(nodes: list[Node]): def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: - def find_parent(a, b) -> Node: + def find_parent(a, b): ia, la = a ib, lb = b min_l = min(la, lb) -- 2.54.0 From 14c8e66f607e6f9e211055d2587c9fab00945503 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 11:07:41 +0300 Subject: [PATCH 85/89] remove unnecessary function --- vpn_manager/cidr4_merge/cidr4_merger.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 5c7fc02..b551ff8 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -48,18 +48,15 @@ def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: mask = (mask << 1) & ((1 << 32) - 1) return ia & mask, min_l - def calc_dip(la, lb, lp): - def dip(l1, lp): - m = lp + 1 - res = 1 << (l1 - m) - res -= 1 - res <<= 32 - l1 - return res - - return dip(la, lp) + dip(lb, lp) + def calc_dip(l1, lp): + m = lp + 1 + res = 1 << (l1 - m) + res -= 1 + res <<= 32 - l1 + return res p = find_parent(a, b) - dip = calc_dip(a[1], b[1], p[1]) + dip = calc_dip(a[1], p[1]) + calc_dip(b[1], p[1]) return p, dip -- 2.54.0 From be9d603bf89d71e5f4f53ef59ef216aaba83805f Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 18:58:17 +0300 Subject: [PATCH 86/89] add individual exceptions --- vpn_manager/cidr4_merge/__init__.py | 2 ++ vpn_manager/cidr4_merge/cidr4_merger.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 vpn_manager/cidr4_merge/__init__.py diff --git a/vpn_manager/cidr4_merge/__init__.py b/vpn_manager/cidr4_merge/__init__.py new file mode 100644 index 0000000..f8176bd --- /dev/null +++ b/vpn_manager/cidr4_merge/__init__.py @@ -0,0 +1,2 @@ +class Cidr4MergerError(Exception): + pass diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index b551ff8..9fda443 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -1,11 +1,17 @@ import cProfile +from vpn_manager.cidr4_merge import Cidr4MergerError + from .util import cidr4_to_node, get_data, make_cidr4 Node = tuple[int, int] -class Cidr4MergerError(Exception): +class EnsureNoSubnetError(Cidr4MergerError): + pass + + +class EnsureNoNeighborsError(Cidr4MergerError): pass @@ -20,7 +26,7 @@ def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: def ensure_no_subnets(nodes: list[Node]): if subnets := find_subnets(nodes): - raise Cidr4MergerError(f"There are subnets! {subnets=}") + raise EnsureNoSubnetError(f"There are subnets! {subnets=}") def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: @@ -34,7 +40,7 @@ def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: def ensure_no_neighbors(nodes: list[Node]): if neighbors := find_neighbors(nodes): - raise Cidr4MergerError(f"There are neighbors! {neighbors=}") + raise EnsureNoNeighborsError(f"There are neighbors! {neighbors=}") def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: -- 2.54.0 From 98583a71014b378ed7543103ba709e7087e94e5e Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 19:03:00 +0300 Subject: [PATCH 87/89] make the algorithm a little faster --- vpn_manager/cidr4_merge/cidr4_merger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py index 9fda443..683f1f7 100644 --- a/vpn_manager/cidr4_merge/cidr4_merger.py +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -74,6 +74,8 @@ def solution(nodes: list[Node], req_len: int) -> tuple[list[Node], int]: p, dip = merge_nodes(a, b) if dip < min_t[2]: min_t = i, p, dip + if dip == 0: + break i, p, dip = min_t nodes = nodes[:i] + [p] + nodes[i + 2 :] sum_dip += dip -- 2.54.0 From 37ffec0558b348c5ac7582cfd148e76162c9600b Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 19:09:09 +0300 Subject: [PATCH 88/89] move util tests to their own file --- tests/cidr4_merge/test_cidr4_merger.py | 27 ----------------------- tests/cidr4_merge/test_util.py | 30 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 27 deletions(-) create mode 100644 tests/cidr4_merge/test_util.py diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index 6a50aab..c4d0251 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -1,7 +1,5 @@ from vpn_manager.cidr4_merge.cidr4_merger import ( - cidr4_to_node, find_neighbors, - make_cidr4, merge_nodes, solution, ) @@ -11,31 +9,6 @@ def test_true(): assert True -def test_cidr4_to_node(): - assert cidr4_to_node("4.78.139.0/24") == (72256256, 24) - assert cidr4_to_node("0.0.0.0/32") == (0, 32) - - assert cidr4_to_node("23.234.30.0/24") == (401219072, 24) - assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19) - assert cidr4_to_node("23.225.141.0/24") == (400657664, 24) - assert cidr4_to_node("31.13.94.0/23") == (520969728, 23) - - assert cidr4_to_node("0.0.0.0/2") == (0, 2) - assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2) - assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2) - assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2) - - -def test_make_cidr4(): - assert make_cidr4(72256256, 24) == "4.78.139.0/24" - assert make_cidr4(0, 32) == "0.0.0.0/32" - - assert make_cidr4(401219072, 24) == "23.234.30.0/24" - assert make_cidr4(2899902464, 19) == "172.217.0.0/19" - assert make_cidr4(400657664, 24) == "23.225.141.0/24" - assert make_cidr4(520969728, 23) == "31.13.94.0/23" - - def test_merge_nodes(): assert merge_nodes((0, 32), (1, 32)) == ((0, 31), 0) assert merge_nodes((0, 32), (2, 32)) == ((0, 30), 2) diff --git a/tests/cidr4_merge/test_util.py b/tests/cidr4_merge/test_util.py new file mode 100644 index 0000000..ea5eeb2 --- /dev/null +++ b/tests/cidr4_merge/test_util.py @@ -0,0 +1,30 @@ +from vpn_manager.cidr4_merge.util import cidr4_to_node, make_cidr4 + + +def test_true(): + assert True + + +def test_cidr4_to_node(): + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24) + assert cidr4_to_node("0.0.0.0/32") == (0, 32) + + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23) + + assert cidr4_to_node("0.0.0.0/2") == (0, 2) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2) + + +def test_make_cidr4(): + assert make_cidr4(72256256, 24) == "4.78.139.0/24" + assert make_cidr4(0, 32) == "0.0.0.0/32" + + assert make_cidr4(401219072, 24) == "23.234.30.0/24" + assert make_cidr4(2899902464, 19) == "172.217.0.0/19" + assert make_cidr4(400657664, 24) == "23.225.141.0/24" + assert make_cidr4(520969728, 23) == "31.13.94.0/23" -- 2.54.0 From 7024503699035d36e7eb09c0bcf032db74ce0641 Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Sun, 26 Jan 2025 19:10:21 +0300 Subject: [PATCH 89/89] add cosmetic changes --- tests/cidr4_merge/test_cidr4_merger.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py index c4d0251..6d636db 100644 --- a/tests/cidr4_merge/test_cidr4_merger.py +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -1,8 +1,4 @@ -from vpn_manager.cidr4_merge.cidr4_merger import ( - find_neighbors, - merge_nodes, - solution, -) +from vpn_manager.cidr4_merge.cidr4_merger import find_neighbors, merge_nodes, solution def test_true(): -- 2.54.0