# Multiple ISP/WLAN

## KatsuoRyuu

Hi Everyone

Im working on my server setting up some multi WLAN connections.

I basically have 2 connections.

The old one (since i only had one) "internet"

the new one currently has its common name - enp0s19

So I have gotten the configuration information from my ISP

the first is

internet:

IP: 133.212.242.2

mask: 255.255.255.0

gateway: 133.212.242.1

enp0s19

IP: 200.181.220.87

mask: 255.255.255.0

gateway: 200.181.220.1

These are the information that i have been given. I want these to run on the same server, the reason i need 2 is because there is a webpage that need to have a seperate IP to be able to work.

So I have tried to do the following:

```

cat /etc/iproute2/rt_tables

> 255   local

> 254   main

> 253   default

> 10 ISP1

> 20 ISP2

ISP1_IFACE="internet"

ISP2_IFACE="enp0s19"

ip route add default dev $ISP1_IFACE table ISP1

ip route add default dev $ISP2_IFACE table ISP2

ip route show

> default via 133.212.242.1 dev internet proto static 

> 10.1.2.0/24 dev br-wordpress proto kernel scope link src 10.1.2.1 

> 10.1.3.0/24 dev br-admin proto kernel scope link src 10.1.3.1 

> 10.1.4.0/24 dev br-balancer proto kernel scope link src 10.1.4.1 

> 10.1.5.0/24 dev br-tools proto kernel scope link src 10.1.5.1 

> 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 

> 172.18.0.0/16 dev br-7bcef0a6bdd9 proto kernel scope link src 172.18.0.1 linkdown 

> 133.212.242.0/24 dev internet proto kernel scope link src 133.212.242.2 

> 200.181.220.0/24 dev enp0s19 proto kernel scope link src 200.181.220.87 

ip rule add fwmark 20 table ISP1 prio 33000

ip rule add fwmark 10 table ISP2 prio 33000

ip rule show

> 0:   from all lookup local 

> 32766:   from all lookup main 

> 32767:   from all lookup default 

> 33000:   from all fwmark 0x14 lookup ISP1 

> 33000:   from all fwmark 0xa lookup ISP2 

iptables -t mangle -A PREROUTING -j CONNMARK --restore-mark

iptables -t mangle -A PREROUTING -m mark ! --mark 0 -j ACCEPT

iptables -t mangle -A PREROUTING -j MARK --set-mark 10

iptables -t mangle -A PREROUTING -m statistic --mode random --probability 0.5 -j MARK --set-mark 20

iptables -t mangle -A PREROUTING -j CONNMARK --save-mark

```

My IpTables looks like the following:

```

# Generated by iptables-save v1.8.0 on Sun Oct 28 05:21:20 2018

*mangle

:PREROUTING ACCEPT [12483:2824480]

:INPUT ACCEPT [38237:7715180]

:FORWARD ACCEPT [2334:736437]

:OUTPUT ACCEPT [30191:8266788]

:POSTROUTING ACCEPT [32528:9003402]

-A PREROUTING -j CONNMARK --restore-mark --nfmask 0xffffffff --ctmask 0xffffffff

-A PREROUTING -m mark ! --mark 0x0 -j ACCEPT

-A PREROUTING -j MARK --set-xmark 0xa/0xffffffff

-A PREROUTING -m statistic --mode random --probability 0.50000000000 -j MARK --set-xmark 0x14/0xffffffff

-A PREROUTING -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff

COMMIT

# Completed on Sun Oct 28 05:21:20 2018

# Generated by iptables-save v1.8.0 on Sun Oct 28 05:21:20 2018

*filter

:INPUT ACCEPT [119:6408]

:FORWARD ACCEPT [1733:914348]

:OUTPUT ACCEPT [41493:11064168]

:DOCKER-USER - [0:0]

:LOGACCEPT - [0:0]

:LOGDROP - [0:0]

:LOGREJECT - [0:0]

-A INPUT -d 10.1.3.2/32 -p tcp -m tcp --dport 9000 -j LOGACCEPT

-A INPUT -i lo -j LOGACCEPT

-A INPUT -i br-wordpress -j LOGACCEPT

-A INPUT -i br-balancer -j LOGACCEPT

-A INPUT -i br-tools -j LOGACCEPT

-A INPUT -i br-admin -j LOGACCEPT

-A INPUT -i docker0 -j LOGACCEPT

-A INPUT -i br-tools -p tcp -m tcp --dport 1006 -j LOGACCEPT

-A INPUT -i internet -p tcp -m tcp --dport 22 -j LOGACCEPT

-A INPUT ! -i br-balancer -p tcp -m tcp --dport 0:1023 -j LOGDROP

-A INPUT ! -i br-balancer -p udp -m udp --dport 0:1023 -j LOGDROP

-A INPUT ! -i br-wordpress -p tcp -m tcp --dport 0:1023 -j LOGDROP

-A INPUT ! -i br-wordpress -p udp -m udp --dport 0:1023 -j LOGDROP

-A FORWARD -j DOCKER-USER

-A FORWARD -d 10.1.0.0/16 -i internet -j LOGACCEPT

-A DOCKER-USER -j RETURN

-A LOGACCEPT -p tcp -j LOG --log-prefix "TCP  LOG ACCEPT: "

-A LOGACCEPT -p udp -j LOG --log-prefix "UDP  LOG ACCEPT: "

-A LOGACCEPT -p icmp -j LOG --log-prefix "ICMP LOG ACCEPT: "

-A LOGACCEPT -f -j LOG --log-prefix "FRAG LOG ACCEPT: "

-A LOGACCEPT -j ACCEPT

-A LOGDROP -p tcp -j LOG --log-prefix "TCP  LOG DROP  : "

-A LOGDROP -p udp -j LOG --log-prefix "UDP  LOG DROP  : "

-A LOGDROP -p icmp -j LOG --log-prefix "ICMP LOG DROP  : "

-A LOGDROP -f -j LOG --log-prefix "FRAG LOG DROP  : "

-A LOGDROP -j DROP

-A LOGREJECT -p tcp -j LOG --log-prefix "TCP  LOG REJECT: "

-A LOGREJECT -p udp -j LOG --log-prefix "UDP  LOG REJECT: "

-A LOGREJECT -p icmp -j LOG --log-prefix "ICMP LOG REJECT: "

-A LOGREJECT -f -j LOG --log-prefix "FRAG LOG REJECT: "

-A LOGREJECT -j DROP

COMMIT

# Completed on Sun Oct 28 05:21:20 2018

# Generated by iptables-save v1.8.0 on Sun Oct 28 05:21:20 2018

*nat

:PREROUTING ACCEPT [14641:3867868]

:INPUT ACCEPT [3248:193280]

:OUTPUT ACCEPT [3049:183103]

:POSTROUTING ACCEPT [0:0]

-A PREROUTING -i internet -p tcp -m tcp --dport 9000 -j DNAT --to-destination 10.1.3.2

-A PREROUTING -i internet -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.1.4.3

-A PREROUTING -i internet -p tcp -m tcp --dport 443 -j DNAT --to-destination 10.1.4.3

-A PREROUTING -i enp0s19 -p tcp -m tcp --dport 80 -j DNAT --to-destination 10.1.4.3

-A PREROUTING -i enp0s19 -p tcp -m tcp --dport 443 -j DNAT --to-destination 10.1.4.3

-A PREROUTING -i internet -p tcp -m tcp --dport 25 -j DNAT --to-destination 10.1.5.249

-A PREROUTING -i internet -p tcp -m tcp --dport 143 -j DNAT --to-destination 10.1.5.249

-A PREROUTING -i internet -p tcp -m tcp --dport 465 -j DNAT --to-destination 10.1.5.249

-A PREROUTING -i internet -p tcp -m tcp --dport 587 -j DNAT --to-destination 10.1.5.249

-A PREROUTING -i internet -p tcp -m tcp --dport 993 -j DNAT --to-destination 10.1.5.249

-A PREROUTING -i internet -p tcp -m tcp --dport 2200 -j DNAT --to-destination 10.1.5.248:2200

-A POSTROUTING -j MASQUERADE

COMMIT

# Completed on Sun Oct 28 05:21:20 2018

```

and the ifconfig intenet|enp0s19:

```

internet: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500

        inet 133.212.242.2  netmask 255.255.255.0  broadcast 133.212.242.255

        inet6 fe80::648d:19ff:fe9a:5fc9  prefixlen 64  scopeid 0x20<link>

        ether 66:8d:19:xx:xx:xx  txqueuelen 1000  (Ethernet)

        RX packets 92829  bytes 8244401 (7.8 MiB)

        RX errors 0  dropped 85  overruns 0  frame 0

        TX packets 7857  bytes 3356687 (3.2 MiB)

        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

enp0s19: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500

        inet 200.181.220.87  netmask 255.255.255.0  broadcast 200.181.220.255

        inet6 fe80::a400:e2ff:fe55:a4f  prefixlen 64  scopeid 0x20<link>

        ether a6:00:e2:xx:xx:xx  txqueuelen 1000  (Ethernet)

        RX packets 61266  bytes 5115759 (4.8 MiB)

        RX errors 0  dropped 0  overruns 0  frame 0

        TX packets 150  bytes 12658 (12.3 KiB)

        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

```

But after this i am still unable to connect to the site using the new IP, the old sites and connections using the old IP still works.

Im not getting anything in the IP tables log file.

Hope anyone can help get the the las part configured, so that i can use the new IP?

----------

## Hu

What exactly did you try that should have used the new address?  What happened instead?  Did the dual-homed system receive the connection?  Did it respond?

----------

## Ralphred

Your 

```
32766:   from all lookup main
```

 is taking precedence over your custom routing tables.

The rest looks fine for 50/50 load balancing, but is going to route to your outgoing packets randomly according to the firewall rule.

You need a higher (re lower number) prio on your mark catching routing tables, and a rule to mark packets destined for you specific website to 20, after the random setting by iptables.

----------

## KatsuoRyuu

Sorry for the late reply to this post, I suddenly got a lot of work to do.

@Hu

So i have a couple of websites on the server, but a new site has a need to be separated by IP as well as domain.

Since this is a very big cluster server i would like to maintain everything on the same system and not need to 

break things up, or install Virtual Machines.

```

      /--------\        /--------\   

      | old IP |        | new IP | 

      \--------/        \--------/        

            |              |

            |              |     

            +--------------+                   

                    |

                    v            

        /----------------------\

        |  Nginx Load Balancer |

        \----------------------/

                    |

                    |

       +------------+-----------+

       |            |           |

       v            v           v

  /---------\  /---------\  /---------\ 

  | WebSite |  | WebSite |  | WebSite | <--- This requires

  \---------/  \---------/  \---------/      IP Seperation

      ^           ^

      |           |

     These two can use

     a shared IP

```

I have currently tried a lot of things but the response i get every time is nothing and I'm getting timeout. It seems that the connection arrives at the computer and gets accepted but from that point on its like it just disappears

A couple of random times i got connection though and then 1-10 min later it just stopped working, Shutting down the enp connection and re configuring it from scratch gives me a connection for a few min again.

Hope that helps?

@Ralphred

Yes,  I have read about this and I'm somewhat sure this is what i need, but cant really wrap my head around it, do you have a good example or some good reads of it?

----------

## Hu

When I said "What exactly", I meant I wanted you to describe your test procedure in enough detail that someone else could replicate it, assuming common Linux knowledge but no knowledge specific to your environment.  What commands did you run that should have pulled data from the secondary server IP address?  When the command failed, at what layer did it fail: IP, TCP, application protocol?  What errors were reported by the application?  When monitoring the network on the secondary server IP, does it see any TCP packets arrive from the client?  If it does, does it answer them correctly?  Does it always answer packets that require an answer, or is it random whether it responds versus drops traffic?

----------

## Ralphred

 *KatsuoRyuu wrote:*   

> do you have a good example or some good reads of it?

 

No, I can't find any source that describes everything you want, only piecemeal.

I'll do it from scratch, using the info about your set-up you have given. There are 3 things we need to do, first is mark new packets originating from specific WAN interfaces, so we can send responses out from the correct interface/address.

```
#flush the mangle chains

iptables -t mangle -F

#create a chain for our marking rules, and add them

iptables -t mangle -N MARKING

iptables -t mangle -A MARKING -m mark ! --mark 0 -m comment --comment "Leave already marked packets alone" -j RETURN

iptables -t mangle -A MARKING -i internet -m comment --comment "Mark packets from internet" -j MARK --set-mark=10

iptables -t mangle -A MARKING -i internet -j RETURN

iptables -t mangle -A MARKING -i enp0s19 -m comment --comment "Mark packets from enp0s19" -j MARK --set-mark=20

iptables -t mangle -A MARKING -i enp0s19 -j RETURN
```

Second thing is to mark randomly, umarked packets that can use either interface

```

iptables -t mangle -A MARKING -m comment --comment "Random marking of packets for dev/enp0s19 routing" -m statistic --mode random --probability 0.5 -j MARK --set-mark=20

iptables -t mangle -A MARKING -m mark --mark 0 -m comment --comment "Mop up unmarked packets for dev/internet routing" -j MARK --set-mark=10

#recover connection marks, apply the rules to the prerouting chain, save the connmarks

iptables -t mangle -A PREROUTING -j CONNMARK --restore-mark

iptables -t mangle -A PREROUTING -j MARKING

iptables -t mangle -A PREROUTING -j CONNMARK --save-mark
```

The last thing is to update the routing tables/rules to use the marks we have set. There are 2 ways to do this*, the easiest is to copy the main table into your marked routing table, and change the default gateway.

```
ip route flush table 20

ip route show table main|grep -v "^default"| while read route;do ip route add $route table 20;done

ip route add default via 200.181.220.1 dev enp0s19 table 20

#apply the lookup table to the rules list

ip rule add lookup 20 fwmark 20
```

Finally, you have static IP's, you should be using SNAT and not MASQUERADE, it has less overhead. Your filter chain policies are all accept, I dunno if this is intentional, if it isn't there is more filtering work to be done before changing it.

*The other way is to remove the "default route" from the main table, add the default routes for tables 10 and 20 respectively (without copying anything), then set the ip rules so you get 

```
# ip rule

0:      from all lookup local 

32766:  from all lookup main 

32767:  from all lookup default 

32768:  from all fwmark 0x14 lookup 20 

32769:  from all lookup 10
```

It's more of a chore to script, but can be better if you can prevent the default route being set in the main table in the first place, and flushing/copying table 20 is inconvenient when you get changes to the main table.

----------

## KatsuoRyuu

 *Ralphred wrote:*   

>  *KatsuoRyuu wrote:*   do you have a good example or some good reads of it? 
> 
> No, I can't find any source that describes everything you want, only piecemeal.
> 
> I'll do it from scratch, using the info about your set-up you have given. There are 3 things we need to do, first is mark new packets originating from specific WAN interfaces, so we can send responses out from the correct interface/address.
> ...

 

Cool looks exactly like what I need, and yes you are right, I still need some filtering, I opened up a lot of stuff to try to find the issue.

I'll return to you as soon as I have tried it.

And sorry for the slow response, work has been hectic lately.

----------

## KatsuoRyuu

Hi Ralphred

Sorry for interrupting you again, and the long time between responses, as work became so hectic.

I tested out your suggestion but it still seems that I'm unable to connect to the second IP address.

After the changes the settings look like the following:

ip route:

```
default via 133.212.242.1 dev internet proto static 

10.1.1.0/24 dev br-afdd1f59523c proto kernel scope link src 10.1.1.1 

10.1.2.0/24 dev br-wordpress proto kernel scope link src 10.1.2.1 

10.1.3.0/24 dev br-admin proto kernel scope link src 10.1.3.1 

10.1.4.0/24 dev br-balancer proto kernel scope link src 10.1.4.1 

10.1.5.0/24 dev br-tools proto kernel scope link src 10.1.5.1 

172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 

172.18.0.0/16 dev br-7bcef0a6bdd9 proto kernel scope link src 172.18.0.1 linkdown 

133.212.242.0/24 dev internet proto kernel scope link src 133.212.242.2 

200.181.220.0/24 dev enp0s19 proto kernel scope link src 200.181.220.87 
```

ip rule:

```
0:   from all lookup local 

32765:   from all fwmark 0x14 lookup ISP2 

32766:   from all lookup main 

32767:   from all lookup default 
```

cat /etc/iproute2/rt_tables:

```
#

# reserved values

#

255   local

254   main

253   default

0   unspec

#

# local

#

#1   inr.ruhep

10 ISP1

20 ISP2
```

iptables-save: 

```
###

#

#  NAT 

#

##########

*nat

:PREROUTING ACCEPT [205980:21756554]

:INPUT ACCEPT [127457:7566745]

:OUTPUT ACCEPT [118233:7093985]

:POSTROUTING ACCEPT [0:0]

:MARKING - [0:0]

:LB_LOG - [0:0]

:MAIL_LOG - [0:0]

:GIT_LOG - [0:0]

#create a chain for our marking rules, and add them 

-A MARKING -m mark ! --mark 0 -m comment --comment "Leave already marked packets alone" -j RETURN

-A MARKING -i internet -m comment --comment "Mark packets from internet" -j MARK --set-mark=10 

-A MARKING -i internet -j RETURN

-A MARKING -i enp0s19 -m comment --comment "Mark packets from enp0s19" -j MARK --set-mark=20 

-A MARKING -i enp0s19 -j RETURN

# Marking Random packages

-A MARKING -m comment --comment "Random marking of packets for dev/enp0s19 routing" -m statistic --mode random --probability 0.5 -j MARK --set-mark=20 

-A MARKING -m mark --mark 0 -m comment --comment "Mop up unmarked packets for dev/internet routing" -j MARK --set-mark=10 

#recover connection marks, apply the rules to the prerouting chain, save the connmarks 

-A PREROUTING -j CONNMARK --restore-mark 

-A PREROUTING -j MARKING 

-A PREROUTING -j CONNMARK --save-mark

# Portainer temporary

-A PREROUTING -i internet -p tcp -m tcp --dport 9000        -j DNAT --to-destination 10.1.3.2

-A PREROUTING -i internet -p tcp -m tcp --dport 25100       -j DNAT --to-destination 10.1.1.2:25100

# HTTP Server - Not accociated sites

#-A PREROUTING -i enp0s19  -p tcp -m tcp --dport 80          -j LB_LOG

#-A PREROUTING -i enp0s19  -p tcp -m tcp --dport 443         -j LB_LOG

# HTTP Server

-A PREROUTING -i internet -p tcp -m tcp --dport 80          -j LB_LOG

-A PREROUTING -i internet -p tcp -m tcp --dport 443         -j LB_LOG

# Mail Server

-A PREROUTING -i internet -p tcp -m tcp --dport 25          -j MAIL_LOG

-A PREROUTING -i internet -p tcp -m tcp --dport 143         -j MAIL_LOG

-A PREROUTING -i internet -p tcp -m tcp --dport 465         -j MAIL_LOG

-A PREROUTING -i internet -p tcp -m tcp --dport 587         -j MAIL_LOG

-A PREROUTING -i internet -p tcp -m tcp --dport 993         -j MAIL_LOG

# GitLab port 22

-A PREROUTING -i internet -p tcp -m tcp --dport 2200        -j GIT_LOG

# Logging Forwarded connection from nat 

# 

-A LB_LOG                                                   -j LOG --log-prefix "==> NAT FORWARD: "

-A LB_LOG                                                   -j DNAT --to-destination 10.1.4.3

-A MAIL_LOG                                                 -j LOG --log-prefix "==> NAT FORWARD: "

-A MAIL_LOG                                                 -j DNAT --to-destination 10.1.5.249

-A GIT_LOG                                                  -j LOG --log-prefix "==> NAT FORWARD: "

-A GIT_LOG                -p tcp -m tcp                     -j DNAT --to-destination 10.1.5.248:22

# Postroute stuff

-A POSTROUTING -o internet                                  -j MASQUERADE

COMMIT

###

#

#  Filters 

#

##########

*filter

:INPUT DROP [6663:322232]

:FORWARD ACCEPT [550395:321585919]

:OUTPUT ACCEPT [1595845:628713431]

:DOCKER-USER - [0:0]

:ACCEPT-IN - [0:0]

:ACCEPT-OUT - [0:0]

:ACCEPT-FWD - [0:0]

:DROP-IN - [0:0]

:REJECT-IN - [0:0]

# Allow the internet connection to connect to internal IPs

#

-A INPUT   -d 10.1.3.2/32    -p tcp -m tcp --dport 9000     -j ACCEPT-IN

#-A INPUT   -i lo                                            -j ACCEPT-IN

# Allow for internal connections

-A INPUT   -s 10.1.1.0/24    -p tcp -m tcp                  -j ACCEPT-IN

-A INPUT   -s 10.1.2.0/24    -p tcp -m tcp                  -j ACCEPT-IN

-A INPUT   -s 10.1.3.0/24    -p tcp -m tcp                  -j ACCEPT-IN

-A INPUT   -s 10.1.4.0/24    -p tcp -m tcp                  -j ACCEPT-IN

-A INPUT   -s 10.1.5.0/24    -p tcp -m tcp                  -j ACCEPT-IN

# GIT ssh port

-A INPUT   -s 10.1.5.0/24    -p tcp -m tcp --dport 2200     -j ACCEPT-IN

# Allow for Web ports

-A INPUT   -i internet       -p tcp -m tcp --dport 80       -j ACCEPT-IN

-A INPUT   -i internet       -p tcp -m tcp --dport 443      -j ACCEPT-IN

# Secondary webpages

-A INPUT   -i enp0s19        -p tcp -m tcp --dport 80       -j ACCEPT-IN

-A INPUT   -i enp0s19        -p tcp -m tcp --dport 443      -j ACCEPT-IN

# Allow for Mail services

-A INPUT   -i internet       -p tcp -m tcp --dport 25       -j ACCEPT-IN

-A INPUT   -i internet       -p tcp -m tcp --dport 143      -j ACCEPT-IN

-A INPUT   -i internet       -p tcp -m tcp --dport 465      -j ACCEPT-IN

-A INPUT   -i internet       -p tcp -m tcp --dport 587      -j ACCEPT-IN

-A INPUT   -i internet       -p tcp -m tcp --dport 993      -j ACCEPT-IN

# Game Servers

-A INPUT   -i internet       -p tcp -m tcp --dport 25110    -j ACCEPT-IN

-A INPUT   -i internet       -p tcp -m tcp --dport 25100    -j ACCEPT-IN

#-A INPUT   -i br-wordpress                                  -j ACCEPT-IN

#-A INPUT   -i br-balancer                                   -j ACCEPT-IN

#-A INPUT   -i br-tools                                      -j ACCEPT-IN

#-A INPUT   -i br-admin                                      -j ACCEPT-IN

#-A INPUT   -i docker0                                       -j ACCEPT-IN

#-A INPUT   -i br-tools       -p tcp -m tcp --dport 1006     -j ACCEPT-IN

-A INPUT   -i internet       -p tcp -m tcp --dport 22       -j ACCEPT-IN

# Rejecting anything with domain from internet

#

-A INPUT -p UDP --dport bootps ! -i br-balancer             -j REJECT-IN

-A INPUT -p UDP --dport domain ! -i br-balancer             -j REJECT-IN

-A INPUT -p UDP --dport bootps ! -i br-wordpress            -j REJECT-IN

-A INPUT -p UDP --dport domain ! -i br-wordpress            -j REJECT-IN

-A INPUT -p UDP --dport bootps ! -i br-tools                -j REJECT-IN

-A INPUT -p UDP --dport domain ! -i br-tools                -j REJECT-IN

-A INPUT -p UDP --dport bootps ! -i br-admin                -j REJECT-IN

-A INPUT -p UDP --dport domain ! -i br-admin                -j REJECT-IN

# Allow the internet connection to connect to internal IPs

#

-A FORWARD -i enp0s19  -d 10.1.0.0/16                       -j ACCEPT-FWD

-A FORWARD -i internet -d 10.1.0.0/16                       -j ACCEPT-FWD

#

# BLOCK EVERYTHING ELSE!

#

#-A INPUT -i internet                                       -j ACCEPT-IN

-A INPUT                                                    -j DROP-IN

-A OUTPUT                                                   -j ACCEPT-OUT

#

# Logging

# 

# Log and accept the connection INPUT

#

-A ACCEPT-IN -p tcp                                         -j LOG --log-prefix "ACCEPT IN  TCP : "

-A ACCEPT-IN -p udp                                         -j LOG --log-prefix "ACCEPT IN  UDP : "

-A ACCEPT-IN -p icmp                                        -j LOG --log-prefix "ACCEPT IN  ICMP: "

-A ACCEPT-IN -f                                             -j LOG --log-prefix "ACCEPT IN  FRAG: "

-A ACCEPT-IN                                                -j ACCEPT

# Log and accept the connection OUTPUT

#

-A ACCEPT-OUT -p tcp                                        -j LOG --log-prefix "ACCEPT OUT TCP : "

-A ACCEPT-OUT -p udp                                        -j LOG --log-prefix "ACCEPT OUT UDP : "

-A ACCEPT-OUT -p icmp                                       -j LOG --log-prefix "ACCEPT OUT ICMP: "

-A ACCEPT-OUT -f                                            -j LOG --log-prefix "ACCEPT OUT FRAG: "

-A ACCEPT-OUT                                               -j ACCEPT

# Log and accept the connection OUTPUT

#

-A ACCEPT-FWD -p tcp                                        -j LOG --log-prefix "ACCEPT FWD TCP : "

-A ACCEPT-FWD -p udp                                        -j LOG --log-prefix "ACCEPT FWD UDP : "

-A ACCEPT-FWD -p icmp                                       -j LOG --log-prefix "ACCEPT FWD ICMP: "

-A ACCEPT-FWD -f                                            -j LOG --log-prefix "ACCEPT FWD FRAG: "

-A ACCEPT-FWD                                               -j ACCEPT

# Log and Drop the connection

#

-A DROP-IN   -p tcp                                         -j LOG --log-prefix "DROP   IN  TCP : "

-A DROP-IN   -p udp                                         -j LOG --log-prefix "DROP   IN  UDP : "

-A DROP-IN   -p icmp                                        -j LOG --log-prefix "DROP   IN  ICMP: "

-A DROP-IN   -f                                             -j LOG --log-prefix "DROP   IN  FRAG: "

-A DROP-IN                                                  -j DROP

# Log and Reject the connection

#

-A REJECT-IN -p tcp                                         -j LOG --log-prefix "REJECT IN  TCP : "

-A REJECT-IN -p udp                                         -j LOG --log-prefix "REJECT IN  UDP : "

-A REJECT-IN -p icmp                                        -j LOG --log-prefix "REJECT IN  ICMP: "

-A REJECT-IN -f                                             -j LOG --log-prefix "REJECT IN  FRAG: "

-A REJECT-IN                                                -j DROP

COMMIT

# Completed on Mon Oct  1 07:52:42 2018
```

I have tried to track whats going on and for some reason the only package mark i see is:

```
Nov 27 07:23:07 AnubisBackend kernel: ACCEPT IN  TCP : IN=br-balancer OUT= MAC=02:42:5c:aa:0a:cd:02:42:0a:01:04:03:08:00 SRC=10.1.4.3 DST=10.1.5.1 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=5296 DF PROTO=TCP SPT=53804 DPT=61005 WINDOW=29200 RES=0x00 SYN URGP=0 MARK=0x14
```

Im running some docker containers internally in the server i dont know if that has any effect on the system. 

Further more i have tried to see if im getting any information upon and according to iptraf-ng (about 4min):

```
               Total      Total    Incoming   Incoming    Outgoing   Outgoing

             Packets      Bytes     Packets      Bytes     Packets      Bytes

 Total:          238      37379         225      36457          13        922    
```

So it seems that i am on the net with it when i try to access the webpage connected to that IP i can see the "Incoming rate" goes up and then dies out with basically nothing on the outgoing 

in comparison to the other connection "internet" this was generates in only a few secs of monitoring:

```
               Total      Total    Incoming   Incoming    Outgoing   Outgoing

             Packets      Bytes     Packets      Bytes     Packets      Bytes

 Total:         4779     382856        1871      95354        2908     287502
```

This connection has an average "Incoming rate" of 1mbps and 2mbps on the outgoing.

The "route" command return this:

```
Kernel IP routing table

Destination     Gateway         Genmask         Flags Metric Ref    Use Iface

default         133.212.242.1   0.0.0.0         UG    0      0        0 internet

10.1.1.0        0.0.0.0         255.255.255.0   U     0      0        0 br-afdd1f59523c

10.1.2.0        0.0.0.0         255.255.255.0   U     0      0        0 br-wordpress

10.1.3.0        0.0.0.0         255.255.255.0   U     0      0        0 br-admin

10.1.4.0        0.0.0.0         255.255.255.0   U     0      0        0 br-balancer

10.1.5.0        0.0.0.0         255.255.255.0   U     0      0        0 br-tools

172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0

172.18.0.0      0.0.0.0         255.255.0.0     U     0      0        0 br-7bcef0a6bdd9

133.212.242.0   0.0.0.0         255.255.255.0   U     0      0        0 internet

200.181.220.0   0.0.0.0         255.255.255.0   U     0      0        0 enp0s19
```

Unfortunately im unaware of what other information i can supply to help, though i so also have a log of all of the stuff from iptables, if you need any of that please let me know what part would be good as it generates large amounts of data (about 6GB a day)

Again i really appreciate the help!

----------

## Ralphred

The MARKING chain is in the nat section, it should be in mangle, all the marking rules should be in mangle. It might work in nat, but I've never tried it outside of mangle except for tracing packets through the firewall.

route on it's own won't show very much with this set-up, you need to use the names from ip rule show with ip route show table [name]

It's always better to use the iptables command to add rules etc, that way is there is a syntax issue it will highlight it, try not to edit /var/lib/iptables/rules-save directly.

----------

## KatsuoRyuu

Hi Raiphred

I took your advice and change my script to look like this:

```

#!/bin/bash

NAT="iptables -t nat"

FILTER="iptables -t filter"

MANGLE="iptables -t mangle"

EXTRA_IF="enp0s19"

EXTRA_IP=`ifconfig ${EXTRA_IF} | grep "inet " | sed -E "s/.*inet ([0-9\.]+).*/\1/g"`

echo "${EXTRA_IF}  :: ${EXTRA_IP}"

WAN_IF="internet"

WAN_IP=`ifconfig ${WAN_IF} | grep "inet " | sed -E "s/.*inet ([0-9\.]+).*/\1/g"`

echo "${WAN_IF} :: ${WAN_IP}"

###########

#

#  Mangle

#

######################

# =====================================================================================================================================

#   CLEAN & CREATE CHAINS

# =====================================================================================================================================

###########

# Cleaning

#

${MANGLE} -F

###########

# Create chains

#

${MANGLE} -N MARKING

# =====================================================================================================================================

#   PACKAGE MARKING

# =====================================================================================================================================

###########

# create a chain for our marking rules, and add them 

#

${MANGLE} -A MARKING -m mark ! --mark 0 -m comment --comment "Leave already marked packets alone" -j RETURN

${MANGLE} -A MARKING -i ${WAN_IF} -m comment --comment "Mark packets from ${WAN_IF}" -j MARK --set-mark=10 

${MANGLE} -A MARKING -i ${WAN_IF} -j RETURN

${MANGLE} -A MARKING -i ${EXTRA_IF} -m comment --comment "Mark packets from ${EXTRA_IF}" -j MARK --set-mark=20 

${MANGLE} -A MARKING -i ${EXTRA_IF} -j RETURN

###########

# Marking Random packages

#

${MANGLE} -A MARKING -m comment --comment "Random marking of packets for dev/${EXTRA_IF} routing" -m statistic --mode random --probability 0.5 -j MARK --set-mark=20 

${MANGLE} -A MARKING -m mark --mark 0 -m comment --comment "Mop up unmarked packets for dev/${WAN_IF} routing" -j MARK --set-mark=10 

###########

# recover connection marks, apply the rules to the prerouting chain, save the connmarks 

#

${MANGLE} -A PREROUTING -j CONNMARK --restore-mark 

${MANGLE} -A PREROUTING -j MARKING 

${MANGLE} -A PREROUTING -j CONNMARK --save-mark

###########

#

#  NAT 

#

######################

# =====================================================================================================================================

#   CLEAN & CREATE CHAINS

# =====================================================================================================================================

###########

# Cleaning

#

${NAT} -F

###########

# Create chains

#

${NAT} -N LOG_LOADBALANCER

${NAT} -N LOG_EXTRA_INTERFACE

${NAT} -N LOG_MAIL

${NAT} -N LOG_GITLAB

${NAT} -N LOG_PORTAINER

${NAT} -N LOG_MINECRAFT_100

${NAT} -N LOG_MINECRAFT_110

# =====================================================================================================================================

#   PRE-ROUTING

# =====================================================================================================================================

# -------------------------------------------------------------------------------------------------------------------------------------

#   EXTRA INTERFACE

# -------------------------------------------------------------------------------------------------------------------------------------

${NAT} -A PREROUTING -i ${EXTRA_IF} -p tcp -m tcp --dport 80                                               -j LOG_EXTRA_INTERFACE

${NAT} -A PREROUTING -i ${EXTRA_IF} -p tcp -m tcp --dport 443                                              -j LOG_EXTRA_INTERFACE

${NAT} -A LOG_EXTRA_INTERFACE                                  -m state --state NEW                    -j LOG --log-prefix "[>] NAT FORWARD: "

${NAT} -A LOG_EXTRA_INTERFACE                                  -m state --state ESTABLISHED            -j LOG --log-prefix "[=] NAT FORWARD: "

${NAT} -A LOG_EXTRA_INTERFACE                                                                          -j LOG --log-prefix "[-] NAT FORWARD: "

${NAT} -A LOG_EXTRA_INTERFACE                                                                          -j DNAT --to-destination 173.212.242.2

# -------------------------------------------------------------------------------------------------------------------------------------

#   FRONTEND

# -------------------------------------------------------------------------------------------------------------------------------------

###########

# Load Balancer

#

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 80                                              -j LOG_LOADBALANCER

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 443                                             -j LOG_LOADBALANCER

${NAT} -A LOG_LOADBALANCER                                     -m state --state NEW                    -j LOG --log-prefix "[>] NAT FORWARD: "

${NAT} -A LOG_LOADBALANCER                                     -m state --state ESTABLISHED            -j LOG --log-prefix "[=] NAT FORWARD: "

${NAT} -A LOG_LOADBALANCER                                                                             -j LOG --log-prefix "[-] NAT FORWARD: "

${NAT} -A LOG_LOADBALANCER                                                                             -j DNAT --to-destination 10.1.4.3

# -------------------------------------------------------------------------------------------------------------------------------------

#   TOOLS

# -------------------------------------------------------------------------------------------------------------------------------------

###########

# Mail Server 

#

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 25                                              -j LOG_MAIL

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 143                                             -j LOG_MAIL

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 465                                             -j LOG_MAIL

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 587                                             -j LOG_MAIL

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 993                                             -j LOG_MAIL

${NAT} -A LOG_MAIL                                             -m state --state NEW                    -j LOG --log-prefix "[>] NAT FORWARD: "

${NAT} -A LOG_MAIL                                             -m state --state ESTABLISHED            -j LOG --log-prefix "[=] NAT FORWARD: "

${NAT} -A LOG_MAIL                                                                                     -j LOG --log-prefix "[-] NAT FORWARD: "

${NAT} -A LOG_MAIL                                                                                     -j DNAT --to-destination 10.1.5.249

###########

# GitLab

#

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 2200                                            -j LOG_GITLAB

${NAT} -A LOG_GITLAB                                           -m state --state NEW                    -j LOG --log-prefix "[>] NAT FORWARD: "

${NAT} -A LOG_GITLAB                                           -m state --state ESTABLISHED            -j LOG --log-prefix "[=] NAT FORWARD: "

${NAT} -A LOG_GITLAB                                                                                   -j LOG --log-prefix "[-] NAT FORWARD: "

${NAT} -A LOG_GITLAB             -p tcp -m tcp                                                         -j DNAT --to-destination 10.1.5.248:22

# -------------------------------------------------------------------------------------------------------------------------------------

#   ADMIN

# -------------------------------------------------------------------------------------------------------------------------------------

###########

# Portainer

#

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 9000                                            -j LOG_PORTAINER

${NAT} -A LOG_PORTAINER                                        -m state --state NEW                    -j LOG --log-prefix "[>] NAT FORWARD: "

${NAT} -A LOG_PORTAINER                                        -m state --state ESTABLISHED            -j LOG --log-prefix "[=] NAT FORWARD: "

${NAT} -A LOG_PORTAINER                                                                                -j LOG --log-prefix "[-] NAT FORWARD: "

${NAT} -A LOG_PORTAINER                                                                                -j DNAT --to-destination 10.1.3.2

# -------------------------------------------------------------------------------------------------------------------------------------

#   GAMES

# -------------------------------------------------------------------------------------------------------------------------------------

###########

# Minecraft 100 (Skyfactory 3)

#

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 25100                                           -j LOG_MINECRAFT_100

${NAT} -A LOG_MINECRAFT_100                                    -m state --state NEW                    -j LOG --log-prefix "[>] NAT FORWARD: "

${NAT} -A LOG_MINECRAFT_100                                    -m state --state ESTABLISHED            -j LOG --log-prefix "[=] NAT FORWARD: "

${NAT} -A LOG_MINECRAFT_100                                                                            -j LOG --log-prefix "[-] NAT FORWARD: "

${NAT} -A LOG_MINECRAFT_100      -p tcp -m tcp                                                         -j DNAT --to-destination 10.1.1.2:25565

###########

# Minecraft 110 (All the Mods 3 Lite)

#

${NAT} -A PREROUTING -i ${WAN_IF} -p tcp -m tcp --dport 25110                                           -j LOG_MINECRAFT_110

${NAT} -A POSTROUTING -p tcp -d 192.168.12.77 --dport 80 -j SNAT --to-source 192.168.12.87

${NAT} -A LOG_MINECRAFT_110                                    -m state --state NEW                    -j LOG --log-prefix "[>] NAT FORWARD: "

${NAT} -A LOG_MINECRAFT_110                                    -m state --state ESTABLISHED            -j LOG --log-prefix "[=] NAT FORWARD: "

${NAT} -A LOG_MINECRAFT_110                                                                            -j LOG --log-prefix "[-] NAT FORWARD: "

${NAT} -A LOG_MINECRAFT_110      -p tcp -m tcp                                                         -j DNAT --to-destination 10.1.1.3:25565

# =====================================================================================================================================

#   POST ROUTING

# =====================================================================================================================================

###########

# Return Masqurading

#

${NAT} -A POSTROUTING -o ${WAN_IF}                                                                      -j MASQUERADE

${NAT} -A POSTROUTING -o ${EXTRA_IF}                                                                       -j MASQUERADE

###########

#

#  Filters 

#

######################

# =====================================================================================================================================

#   CLEAN & CREATE CHAINS

# =====================================================================================================================================

###########

# Cleaning

#

${FILTER} -F

###########

# Create chains

#

${FILTER} -N DOCKER-USER

${FILTER} -N ACCEPT-IN

${FILTER} -N ACCEPT-OUT

${FILTER} -N ACCEPT-FWD

${FILTER} -N DROP-IN

${FILTER} -N REJECT-IN

${FILTER} -N DROP-FWD

${FILTER} -N ACCEPT-FWD-E

# Allow the ${WAN_IF} connection to connect to internal IPs

#

${FILTER} -A INPUT   -d 10.1.3.2/32    -p tcp -m tcp --dport 9000                                      -j ACCEPT-IN

#${FILTER} -A INPUT   -i lo                                                                             -j ACCEPT-IN

# Allow for internal connections

${FILTER} -A INPUT   -s 10.1.1.0/24    -p tcp -m tcp                                                   -j ACCEPT-IN

${FILTER} -A INPUT   -s 10.1.2.0/24    -p tcp -m tcp                                                   -j ACCEPT-IN

${FILTER} -A INPUT   -s 10.1.3.0/24    -p tcp -m tcp                                                   -j ACCEPT-IN

${FILTER} -A INPUT   -s 10.1.4.0/24    -p tcp -m tcp                                                   -j ACCEPT-IN

${FILTER} -A INPUT   -s 10.1.5.0/24    -p tcp -m tcp                                                   -j ACCEPT-IN

# GIT ssh port

${FILTER} -A INPUT   -s 10.1.5.0/24    -p tcp -m tcp --dport 2200                                      -j ACCEPT-IN

# Allow for Web ports

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 80                                        -j ACCEPT-IN

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 443                                       -j ACCEPT-IN

# Secondary webpages

${FILTER} -A INPUT   -i ${EXTRA_IF}        -p tcp -m tcp --dport 80                                        -j ACCEPT-IN

${FILTER} -A INPUT   -i ${EXTRA_IF}        -p tcp -m tcp --dport 443                                       -j ACCEPT-IN

# Allow for Mail services

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 25                                        -j ACCEPT-IN

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 143                                       -j ACCEPT-IN

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 465                                       -j ACCEPT-IN

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 587                                       -j ACCEPT-IN

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 993                                       -j ACCEPT-IN

# Game Servers

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 25110                                     -j ACCEPT-IN

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 25100                                     -j ACCEPT-IN

#${FILTER} -A INPUT   -i br-wordpress                                                                   -j ACCEPT-IN

#${FILTER} -A INPUT   -i br-balancer                                                                    -j ACCEPT-IN

#${FILTER} -A INPUT   -i br-tools                                                                       -j ACCEPT-IN

#${FILTER} -A INPUT   -i br-admin                                                                       -j ACCEPT-IN

#${FILTER} -A INPUT   -i docker0                                                                        -j ACCEPT-IN

#${FILTER} -A INPUT   -i br-tools       -p tcp -m tcp --dport 1006                                      -j ACCEPT-IN

${FILTER} -A INPUT   -i ${WAN_IF}       -p tcp -m tcp --dport 22                                        -j ACCEPT-IN

# Rejecting anything with domain from ${WAN_IF}

#

${FILTER} -A INPUT -p UDP --dport bootps ! -i br-balancer                                              -j REJECT-IN

${FILTER} -A INPUT -p UDP --dport domain ! -i br-balancer                                              -j REJECT-IN

${FILTER} -A INPUT -p UDP --dport bootps ! -i br-wordpress                                             -j REJECT-IN

${FILTER} -A INPUT -p UDP --dport domain ! -i br-wordpress                                             -j REJECT-IN

${FILTER} -A INPUT -p UDP --dport bootps ! -i br-tools                                                 -j REJECT-IN

${FILTER} -A INPUT -p UDP --dport domain ! -i br-tools                                                 -j REJECT-IN

${FILTER} -A INPUT -p UDP --dport bootps ! -i br-admin                                                 -j REJECT-IN

${FILTER} -A INPUT -p UDP --dport domain ! -i br-admin                                                 -j REJECT-IN

# Allow the ${WAN_IF} connection to connect to internal IPs

#

${FILTER} -A FORWARD -i ${EXTRA_IF}  -d 10.1.0.0/16                                                        -j ACCEPT-FWD

${FILTER} -A FORWARD -i ${WAN_IF} -d 10.1.0.0/16                                                        -j ACCEPT-FWD

${FILTER} -A FORWARD                                                                                   -j DROP-FWD

#

# BLOCK EVERYTHING ELSE!

#

#-A INPUT -i ${WAN_IF}                                                                                  -j ACCEPT-IN

${FILTER} -A INPUT                                                                                     -j DROP-IN

${FILTER} -A OUTPUT                                                                                    -j ACCEPT-OUT

#

# Logging

# 

# Log and accept the connection INPUT

#

${FILTER} -A ACCEPT-IN -p tcp                                                                          -j LOG --log-prefix "ACCEPT IN  TCP : "

${FILTER} -A ACCEPT-IN -p udp                                                                          -j LOG --log-prefix "ACCEPT IN  UDP : "

${FILTER} -A ACCEPT-IN -p icmp                                                                         -j LOG --log-prefix "ACCEPT IN  ICMP: "

${FILTER} -A ACCEPT-IN -f                                                                              -j LOG --log-prefix "ACCEPT IN  FRAG: "

${FILTER} -A ACCEPT-IN                                                                                 -j ACCEPT

# Log and accept the connection OUTPUT

#

${FILTER} -A ACCEPT-OUT -p tcp                                                                         -j LOG --log-prefix "ACCEPT OUT TCP : "

${FILTER} -A ACCEPT-OUT -p udp                                                                         -j LOG --log-prefix "ACCEPT OUT UDP : "

${FILTER} -A ACCEPT-OUT -p icmp                                                                        -j LOG --log-prefix "ACCEPT OUT ICMP: "

${FILTER} -A ACCEPT-OUT -f                                                                             -j LOG --log-prefix "ACCEPT OUT FRAG: "

${FILTER} -A ACCEPT-OUT                                                                                -j ACCEPT

# Log and accept the connection OUTPUT

#

${FILTER} -A ACCEPT-FWD -p tcp                                                                         -j LOG --log-prefix "ACCEPT FWD TCP : "

${FILTER} -A ACCEPT-FWD -p udp                                                                         -j LOG --log-prefix "ACCEPT FWD UDP : "

${FILTER} -A ACCEPT-FWD -p icmp                                                                        -j LOG --log-prefix "ACCEPT FWD ICMP: "

${FILTER} -A ACCEPT-FWD -f                                                                             -j LOG --log-prefix "ACCEPT FWD FRAG: "

${FILTER} -A ACCEPT-FWD                                                                                -j ACCEPT

# Log and accept the ESTABLISHED/RELATED connection FORWARD 

#

${FILTER} -A ACCEPT-FWD-E -p tcp                                                                       -j LOG --log-prefix "ACCEPT FW+ TCP : "

${FILTER} -A ACCEPT-FWD-E -p udp                                                                       -j LOG --log-prefix "ACCEPT FW+ UDP : "

${FILTER} -A ACCEPT-FWD-E -p icmp                                                                      -j LOG --log-prefix "ACCEPT FW+ ICMP: "

${FILTER} -A ACCEPT-FWD-E -f                                                                           -j LOG --log-prefix "ACCEPT FW+ FRAG: "

${FILTER} -A ACCEPT-FWD-E                                                                              -j ACCEPT

# Log and Drop the connection

#

${FILTER} -A DROP-IN   -p tcp                                                                          -j LOG --log-prefix "DROP   IN  TCP : "

${FILTER} -A DROP-IN   -p udp                                                                          -j LOG --log-prefix "DROP   IN  UDP : "

${FILTER} -A DROP-IN   -p icmp                                                                         -j LOG --log-prefix "DROP   IN  ICMP: "

${FILTER} -A DROP-IN   -f                                                                              -j LOG --log-prefix "DROP   IN  FRAG: "

${FILTER} -A DROP-IN                                                                                   -j DROP

# Log and Reject the connection

#

${FILTER} -A REJECT-IN -p tcp                                                                          -j LOG --log-prefix "REJECT IN  TCP : "

${FILTER} -A REJECT-IN -p udp                                                                          -j LOG --log-prefix "REJECT IN  UDP : "

${FILTER} -A REJECT-IN -p icmp                                                                         -j LOG --log-prefix "REJECT IN  ICMP: "

${FILTER} -A REJECT-IN -f                                                                              -j LOG --log-prefix "REJECT IN  FRAG: "

${FILTER} -A REJECT-IN                                                                                 -j DROP

# Log and accept the connection OUTPUT

#

${FILTER} -A DROP-FWD -p tcp                                                                           -j LOG --log-prefix "DROP   FWD TCP : "

${FILTER} -A DROP-FWD -p udp                                                                           -j LOG --log-prefix "DROP   FWD UDP : "

${FILTER} -A DROP-FWD -p icmp                                                                          -j LOG --log-prefix "DROP   FWD ICMP: "

${FILTER} -A DROP-FWD -f                                                                               -j LOG --log-prefix "DROP   FWD FRAG: "

${FILTER} -A DROP-FWD                                                                                  -j DROP

###########

#

#  IP Route/Rule Adjustment

#

######################

ip route flush table 20 

ip route show table main | grep -v "^default" | while read route; do ip route add $route table 20; done 

ip route add default via ${EXTRA_IP} dev ${EXTRA_IF} table 20

#apply the lookup table to the rules list 

ip rule add lookup 20 fwmark 20

```

```

AnubisBackend ~ # ip rule

0:      from all lookup local 

32763:  from all fwmark 0x14 lookup ISP2 

32764:  from all fwmark 0x14 lookup ISP2 

32765:  from all fwmark 0x14 lookup ISP2 

32766:  from all lookup main 

32767:  from all lookup default 

33000:  from all fwmark 0x14 lookup ISP1 

33000:  from all fwmark 0xa lookup ISP2 

AnubisBackend ~ # ip route show table ISP1

default dev internet scope link 

AnubisBackend ~ # ip route show table ISP2

default via 207.180.226.87 dev enp0s19 

10.1.1.0/24 dev br-afdd1f59523c proto kernel scope link src 10.1.1.1 

10.1.2.0/24 dev br-wordpress proto kernel scope link src 10.1.2.1 

10.1.3.0/24 dev br-admin proto kernel scope link src 10.1.3.1 

10.1.4.0/24 dev br-balancer proto kernel scope link src 10.1.4.1 

10.1.5.0/24 dev br-tools proto kernel scope link src 10.1.5.1 

133.212.242.0/24 dev internet proto kernel scope link src 133.212.242.2

200.181.220.0/24 dev enp0s19 proto kernel scope link src 200.181.220.87

AnubisBackend ~ # ip route show table 10

default dev internet scope link

AnubisBackend ~ # ip route show table 20

default via 207.180.226.87 dev enp0s19 

10.1.1.0/24 dev br-afdd1f59523c proto kernel scope link src 10.1.1.1 

10.1.2.0/24 dev br-wordpress proto kernel scope link src 10.1.2.1 

10.1.3.0/24 dev br-admin proto kernel scope link src 10.1.3.1 

10.1.4.0/24 dev br-balancer proto kernel scope link src 10.1.4.1 

10.1.5.0/24 dev br-tools proto kernel scope link src 10.1.5.1 

133.212.242.0/24 dev internet proto kernel scope link src 133.212.242.2

200.181.220.0/24 dev enp0s19 proto kernel scope link src 200.181.220.87

```

----------

