From 6d718ec9877c3ecfc10681e85d050d15ed4d4b80 Mon Sep 17 00:00:00 2001
From: Christian Hesse <mail@eworm.de>
Date: Thu, 6 Mar 2025 14:38:35 +0100
Subject: [PATCH 1/7] fw-addr-lists: use $LogPrintVerbose ...

... to reduce debug output and speed up execution.
---
 fw-addr-lists.rsc | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index d41dc04..af8c65d 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -24,6 +24,7 @@
   :global HumanReadableNum;
   :global LogPrint;
   :global LogPrintOnce;
+  :global LogPrintVerbose;
   :global ScriptLock;
   :global WaitFullyConnected;
 
@@ -120,14 +121,14 @@
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ip/firewall/address-list/get $Entry address ];
       :if ([ :typeof ($IPv4Addresses->$Address) ] = "time") do={
-        $LogPrint debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
+        $LogPrintVerbose debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
             "' with " . ($IPv4Addresses->$Address) . ": " . $Address);
         /ip/firewall/address-list/set $Entry timeout=($IPv4Addresses->$Address);
         :set ($IPv4Addresses->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={
         :if ($Failure = false) do={
-          $LogPrint debug $ScriptName ("Removing IPv4 address from list '" . $FwListName . \
+          $LogPrintVerbose debug $ScriptName ("Removing IPv4 address from list '" . $FwListName . \
               "': " . $Address);
           /ip/firewall/address-list/remove $Entry;
           :set CntRemove ($CntRemove + 1);
@@ -139,14 +140,14 @@
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ipv6/firewall/address-list/get $Entry address ];
       :if ([ :typeof ($IPv6Addresses->$Address) ] = "time") do={
-        $LogPrint debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \
+        $LogPrintVerbose debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \
             "' with " . ($IPv6Addresses->$Address) . ": " . $Address);
         /ipv6/firewall/address-list/set $Entry timeout=($IPv6Addresses->$Address);
         :set ($IPv6Addresses->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={
         :if ($Failure = false) do={
-          $LogPrint debug $ScriptName ("Removing IPv6 address from list '" . $FwListName . \
+          $LogPrintVerbose debug $ScriptName ("Removing IPv6 address from list '" . $FwListName . \
               "': " . $Address);
           /ipv6/firewall/address-list/remove $Entry;
           :set CntRemove ($CntRemove + 1);
@@ -155,7 +156,7 @@
     }
 
     :foreach Address,Timeout in=$IPv4Addresses do={
-      $LogPrint debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \
+      $LogPrintVerbose debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \
           "' with " . $Timeout . ": " . $Address);
       :do {
         /ip/firewall/address-list/add list=$FwListName comment=$ListComment \
@@ -169,7 +170,7 @@
     }
 
     :foreach Address,Timeout in=$IPv6Addresses do={
-      $LogPrint debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \
+      $LogPrintVerbose debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \
           "' with " . $Timeout . ": " . $Address);
       :do {
         /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment \

From fb343c99e399d679f3e2a22e76c14a5899024319 Mon Sep 17 00:00:00 2001
From: Christian Hesse <mail@eworm.de>
Date: Thu, 6 Mar 2025 22:59:31 +0100
Subject: [PATCH 2/7] fw-addr-lists: put timeout into variable

---
 fw-addr-lists.rsc | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index af8c65d..331d2c6 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -120,10 +120,11 @@
     :foreach Entry in=[ /ip/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ip/firewall/address-list/get $Entry address ];
-      :if ([ :typeof ($IPv4Addresses->$Address) ] = "time") do={
+      :local TimeOut ($IPv4Addresses->$Address);
+      :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
-            "' with " . ($IPv4Addresses->$Address) . ": " . $Address);
-        /ip/firewall/address-list/set $Entry timeout=($IPv4Addresses->$Address);
+            "' with " . $TimeOut . ": " . $Address);
+        /ip/firewall/address-list/set $Entry timeout=$TimeOut;
         :set ($IPv4Addresses->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={
@@ -139,10 +140,11 @@
     :foreach Entry in=[ /ipv6/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ipv6/firewall/address-list/get $Entry address ];
-      :if ([ :typeof ($IPv6Addresses->$Address) ] = "time") do={
+      :local TimeOut ($IPv6Addresses->$Address);
+      :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \
-            "' with " . ($IPv6Addresses->$Address) . ": " . $Address);
-        /ipv6/firewall/address-list/set $Entry timeout=($IPv6Addresses->$Address);
+            "' with " . $TimeOut . ": " . $Address);
+        /ipv6/firewall/address-list/set $Entry timeout=$TimeOut;
         :set ($IPv6Addresses->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={

From ea6de35699a5b62dfffcc583c1a7181e1a6a688f Mon Sep 17 00:00:00 2001
From: Christian Hesse <mail@eworm.de>
Date: Thu, 6 Mar 2025 23:00:50 +0100
Subject: [PATCH 3/7] fw-addr-lists: do not clean up

Cleanup is important on renew (so the script does not attempt to re-add),
but we do not care here.
---
 fw-addr-lists.rsc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index 331d2c6..70893e3 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -163,7 +163,6 @@
       :do {
         /ip/firewall/address-list/add list=$FwListName comment=$ListComment \
             address=$Address timeout=$Timeout;
-        :set ($IPv4Addresses->$Address);
         :set CntAdd ($CntAdd + 1);
       } on-error={
         $LogPrint warning $ScriptName ("Failed to add IPv4 address to list '" . $FwListName . \
@@ -177,7 +176,6 @@
       :do {
         /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment \
             address=$Address timeout=$Timeout;
-        :set ($IPv6Addresses->$Address);
         :set CntAdd ($CntAdd + 1);
       } on-error={
         $LogPrint warning $ScriptName ("Failed to add IPv6 address to list '" . $FwListName . \

From 2f55bfaf009d726e1174b50887fdb8f9b4b57573 Mon Sep 17 00:00:00 2001
From: Christian Hesse <mail@eworm.de>
Date: Fri, 7 Mar 2025 12:21:22 +0100
Subject: [PATCH 4/7] fw-addr-lists: strip cidr for host addresses

This makes sure the addresses match later when we read them from
address-list for renew.
---
 fw-addr-lists.rsc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index 70893e3..fc53795 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -101,10 +101,16 @@
         }
         :do {
           :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$") do={
+            :if ($Address ~ "/32\$") do={
+              :set Address [ :pick $Address 0 ([ :len $Address ] - 3) ];
+            }
             :set ($IPv4Addresses->$Address) $TimeOut;
             :error true;
           }
           :if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$") do={
+            :if ($Address ~ "/128\$") do={
+              :set Address [ :pick $Address 0 ([ :len $Address ] - 4) ];
+            }
             :set ($IPv6Addresses->$Address) $TimeOut;
             :error true;
           }

From e148df9e573801803e35d8dfaef277f571952fdc Mon Sep 17 00:00:00 2001
From: Christian Hesse <mail@eworm.de>
Date: Thu, 6 Mar 2025 23:05:45 +0100
Subject: [PATCH 5/7] fw-addr-lists: put addresses into "branches"...

... effectively adding another layer and some complexity, but:
The addresses are sorted inside the array, and sorting less addresses in
a branch saves a lot of processing power. So this is a lot faster now...
---
 fw-addr-lists.rsc | 65 +++++++++++++++++++++++++++--------------------
 1 file changed, 37 insertions(+), 28 deletions(-)

diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index fc53795..a97f7ed 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -100,23 +100,24 @@
           :set Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr"));
         }
         :do {
+          :local Branch [ :pick $Address 0 1 ];
           :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$") do={
             :if ($Address ~ "/32\$") do={
               :set Address [ :pick $Address 0 ([ :len $Address ] - 3) ];
             }
-            :set ($IPv4Addresses->$Address) $TimeOut;
+            :set ($IPv4Addresses->$Branch->$Address) $TimeOut;
             :error true;
           }
           :if ($Address ~ "^[0-9a-zA-Z]*:[0-9a-zA-Z:\\.]+(/[0-9]{1,3})?\$") do={
             :if ($Address ~ "/128\$") do={
               :set Address [ :pick $Address 0 ([ :len $Address ] - 4) ];
             }
-            :set ($IPv6Addresses->$Address) $TimeOut;
+            :set ($IPv6Addresses->$Branch->$Address) $TimeOut;
             :error true;
           }
           :if ($Address ~ "^[\\.a-zA-Z0-9-]+\\.[a-zA-Z]{2,}\$") do={
-            :set ($IPv4Addresses->$Address) $TimeOut;
-            :set ($IPv6Addresses->$Address) $TimeOut;
+            :set ($IPv4Addresses->$Branch->$Address) $TimeOut;
+            :set ($IPv6Addresses->$Branch->$Address) $TimeOut;
             :error true;
           }
         } on-error={ }
@@ -126,12 +127,13 @@
     :foreach Entry in=[ /ip/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ip/firewall/address-list/get $Entry address ];
-      :local TimeOut ($IPv4Addresses->$Address);
+      :local Branch [ :pick $Address 0 1 ];
+      :local TimeOut ($IPv4Addresses->$Branch->$Address);
       :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
             "' with " . $TimeOut . ": " . $Address);
         /ip/firewall/address-list/set $Entry timeout=$TimeOut;
-        :set ($IPv4Addresses->$Address);
+        :set ($IPv4Addresses->$Branch->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={
         :if ($Failure = false) do={
@@ -146,12 +148,13 @@
     :foreach Entry in=[ /ipv6/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ipv6/firewall/address-list/get $Entry address ];
-      :local TimeOut ($IPv6Addresses->$Address);
+      :local Branch [ :pick $Address 0 1 ];
+      :local TimeOut ($IPv6Addresses->$Branch->$Address);
       :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \
             "' with " . $TimeOut . ": " . $Address);
         /ipv6/firewall/address-list/set $Entry timeout=$TimeOut;
-        :set ($IPv6Addresses->$Address);
+        :set ($IPv6Addresses->$Branch->$Address);
         :set CntRenew ($CntRenew + 1);
       } else={
         :if ($Failure = false) do={
@@ -163,29 +166,35 @@
       }
     }
 
-    :foreach Address,Timeout in=$IPv4Addresses do={
-      $LogPrintVerbose debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \
-          "' with " . $Timeout . ": " . $Address);
-      :do {
-        /ip/firewall/address-list/add list=$FwListName comment=$ListComment \
-            address=$Address timeout=$Timeout;
-        :set CntAdd ($CntAdd + 1);
-      } on-error={
-        $LogPrint warning $ScriptName ("Failed to add IPv4 address to list '" . $FwListName . \
-            "': " . $Address);
+    :foreach BranchName,Branch in=$IPv4Addresses do={
+      $LogPrintVerbose debug $ScriptName ("Handling branch: " . $BranchName);
+      :foreach Address,Timeout in=$Branch do={
+        $LogPrintVerbose debug $ScriptName ("Adding IPv4 address to list '" . $FwListName . \
+            "' with " . $Timeout . ": " . $Address);
+        :do {
+          /ip/firewall/address-list/add list=$FwListName comment=$ListComment \
+              address=$Address timeout=$Timeout;
+          :set CntAdd ($CntAdd + 1);
+        } on-error={
+          $LogPrint warning $ScriptName ("Failed to add IPv4 address to list '" . $FwListName . \
+              "': " . $Address);
+        }
       }
     }
 
-    :foreach Address,Timeout in=$IPv6Addresses do={
-      $LogPrintVerbose debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \
-          "' with " . $Timeout . ": " . $Address);
-      :do {
-        /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment \
-            address=$Address timeout=$Timeout;
-        :set CntAdd ($CntAdd + 1);
-      } on-error={
-        $LogPrint warning $ScriptName ("Failed to add IPv6 address to list '" . $FwListName . \
-            "': " . $Address);
+    :foreach BranchName,Branch in=$IPv6Addresses do={
+      $LogPrintVerbose debug $ScriptName ("Handling branch: " . $BranchName);
+      :foreach Address,Timeout in=$Branch do={
+        $LogPrintVerbose debug $ScriptName ("Adding IPv6 address to list '" . $FwListName . \
+            "' with " . $Timeout . ": " . $Address);
+        :do {
+          /ipv6/firewall/address-list/add list=$FwListName comment=$ListComment \
+              address=$Address timeout=$Timeout;
+          :set CntAdd ($CntAdd + 1);
+        } on-error={
+          $LogPrint warning $ScriptName ("Failed to add IPv6 address to list '" . $FwListName . \
+              "': " . $Address);
+        }
       }
     }
 

From d71ea804b077d35183df318fb305b213fb961d27 Mon Sep 17 00:00:00 2001
From: Christian Hesse <mail@eworm.de>
Date: Fri, 7 Mar 2025 20:35:36 +0100
Subject: [PATCH 6/7] fw-addr-lists: two characters for branch

Using one character for IPv4 is ok (1 to 9), but IPv6 global unicase
(2000::/3) results in just two different characters (2 and 3).

So let's use first two characters...
---
 fw-addr-lists.rsc | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index a97f7ed..9ad5b3a 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -100,7 +100,7 @@
           :set Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr"));
         }
         :do {
-          :local Branch [ :pick $Address 0 1 ];
+          :local Branch [ :pick $Address 0 2 ];
           :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$") do={
             :if ($Address ~ "/32\$") do={
               :set Address [ :pick $Address 0 ([ :len $Address ] - 3) ];
@@ -127,7 +127,7 @@
     :foreach Entry in=[ /ip/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ip/firewall/address-list/get $Entry address ];
-      :local Branch [ :pick $Address 0 1 ];
+      :local Branch [ :pick $Address 0 2 ];
       :local TimeOut ($IPv4Addresses->$Branch->$Address);
       :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
@@ -148,7 +148,7 @@
     :foreach Entry in=[ /ipv6/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ipv6/firewall/address-list/get $Entry address ];
-      :local Branch [ :pick $Address 0 1 ];
+      :local Branch [ :pick $Address 0 2 ];
       :local TimeOut ($IPv6Addresses->$Branch->$Address);
       :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \

From f0e6cbcfe1a0be05f30d37cd495d4dbea0888f07 Mon Sep 17 00:00:00 2001
From: Christian Hesse <mail@eworm.de>
Date: Sat, 8 Mar 2025 10:09:51 +0100
Subject: [PATCH 7/7] fw-addr-lists: get branch from calculated checksum

The addresses were spread very uneven before.

Let's calculate a checksum, and take the first two characters of that.
The addresses are now spread evenly on 256 branches (0x00 to 0xff).
---
 fw-addr-lists.rsc | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/fw-addr-lists.rsc b/fw-addr-lists.rsc
index 9ad5b3a..f0940fe 100644
--- a/fw-addr-lists.rsc
+++ b/fw-addr-lists.rsc
@@ -37,6 +37,11 @@
     }
   }
 
+  :local GetBranch do={
+    :global EitherOr;
+    :return [ :pick [ :convert transform=md5 to=hex [ :pick $1 0 [ $EitherOr [ :find $1 "/" ] [ :len $1 ] ] ] ] 0 2 ];
+  }
+
   :if ([ $ScriptLock $ScriptName ] = false) do={
     :set ExitOK true;
     :error false;
@@ -100,7 +105,7 @@
           :set Address ([ :pick $Line 0 [ $FindDelim $Line ] ] . ($List->"cidr"));
         }
         :do {
-          :local Branch [ :pick $Address 0 2 ];
+          :local Branch [ $GetBranch $Address ];
           :if ($Address ~ "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}(/[0-9]{1,2})?\$") do={
             :if ($Address ~ "/32\$") do={
               :set Address [ :pick $Address 0 ([ :len $Address ] - 3) ];
@@ -127,7 +132,7 @@
     :foreach Entry in=[ /ip/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ip/firewall/address-list/get $Entry address ];
-      :local Branch [ :pick $Address 0 2 ];
+      :local Branch [ $GetBranch $Address ];
       :local TimeOut ($IPv4Addresses->$Branch->$Address);
       :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv4 address in list '" . $FwListName . \
@@ -148,7 +153,7 @@
     :foreach Entry in=[ /ipv6/firewall/address-list/find where \
         list=$FwListName comment=$ListComment ] do={
       :local Address [ /ipv6/firewall/address-list/get $Entry address ];
-      :local Branch [ :pick $Address 0 2 ];
+      :local Branch [ $GetBranch $Address ];
       :local TimeOut ($IPv6Addresses->$Branch->$Address);
       :if ([ :typeof $TimeOut ] = "time") do={
         $LogPrintVerbose debug $ScriptName ("Renewing IPv6 address in list '" . $FwListName . \