Category: openbsd

Upgrading my firewall to OpenBSD 5.3 required updating my “multihoming-with-single-ISP” patch (see previous and original posts), as dhclient-script is no longer used. Instead, there is a new file, kroute.c. Luckily, moving the functionality from the shell script into C code was quite straightforward.

Here is the current patch to /usr/src/sbin/dhclient:

--- dhclient.c.orig	2013-08-02 10:17:29.000000000 +0300
+++ dhclient.c	2013-08-02 10:17:29.000000000 +0300
@@ -858,7 +858,10 @@
 	client->xid = arc4random();
 	make_request(client->active);
 
-	if (client->active->options[DHO_DHCP_SERVER_IDENTIFIER].len == 4) {
+	/*if (client->active->options[DHO_DHCP_SERVER_IDENTIFIER].len == 4) {*/
+	if (0 && client->active->options[DHO_DHCP_SERVER_IDENTIFIER].len == 4) {
 		memcpy(&client->destination.s_addr,
 		    client->active->options[DHO_DHCP_SERVER_IDENTIFIER].data,
 		    client->active->options[DHO_DHCP_SERVER_IDENTIFIER].len);
--- kroute.c.orig	2013-08-02 10:17:29.000000000 +0300
+++ kroute.c	2013-08-02 20:20:42.000000000 +0300
@@ -256,6 +256,9 @@
 	struct sockaddr_rtlabel label;
 	struct iovec iov[5];
 	int s, len, i, iovcnt = 0;
+	char buf[256];
 
 	/*
 	 * Add a default route via the specified address.
@@ -339,6 +342,17 @@
 	iov[iovcnt].iov_base = &label;
 	iov[iovcnt++].iov_len = sizeof(label);
 
+	/* Update next hop to pf route-to rules */
+	snprintf(buf, 256, "/sbin/pfctl -t gw_%s -T flush", ifi->name);
+	if (system(buf))
+		warning("failed to flush pf table: %s", strerror(errno));
+	snprintf(buf, 256, "/sbin/pfctl -t gw_%s -T add %s", ifi->name,
+		inet_ntoa(gateway.sin_addr));
+	if (system(buf))
+		warning("failed to add to pf table: %s", strerror(errno));
+
 	/* Check for EEXIST since other dhclient may not be done. */
 	for (i = 0; i < 5; i++) {
 		if (writev(s, iov, iovcnt) != -1)

Here is the patch to /etc/rc:

--- rc.orig	2013-08-02 10:17:29.000000000 +0300
+++ rc	2013-08-02 10:17:29.000000000 +0300
@@ -357,6 +357,14 @@
 	mv -f /etc/resolv.conf.save /etc/resolv.conf
 	touch /etc/resolv.conf
 fi
+
+# Allow em0 to receive vlan packets with different MAC addresses
+ifconfig em0 up
+ifconfig bridge0 create
+brconfig bridge0 add em0
+
 . /etc/netstart
 echo rekey > /dev/arandom	# any write triggers an RC4 rekey
 
@@ -370,6 +378,16 @@
 	fi
 fi
 
+# Initialise next hops for pf's route-to rules
+pfctl -t gw_vlan201 -T add \
+    `netstat -f inet -rn | grep default | grep vlan201 | awk '{print $2}'`
+pfctl -t gw_vlan202 -T add \
+    `netstat -f inet -T1 -rn | grep default | grep vlan202 | awk '{print $2}'`
+pfctl -t gw_vlan203 -T add \
+    `netstat -f inet -T2 -rn | grep default | grep vlan203 | awk '{print $2}'`
+
 mount -s /usr >/dev/null 2>&1
 mount -s /var >/dev/null 2>&1

And finally, this is a working /etc/pf.conf:

###############################################################################
# Macros
###############################################################################

if_int      = "re0"
if_ext1     = "vlan201"
if_ext2     = "vlan202"
if_ext3     = "vlan203"
if_extv6    = "gif0"
all_ifs     = "{" $if_int $if_ext1 $if_ext2 $if_ext3 $if_extv6 "}"
ext_ifs     = "{" $if_ext1 $if_ext2 $if_ext3 $if_extv6 "}"
ext_ifs_v4  = "{" $if_ext1 $if_ext2 $if_ext3 "}"
ext_ifs_v6  = "{" $if_extv6 "}"

if_int_v4   = "10.0.0.xx"
home_net_v4 = "10.0.0.0/24"
if_int_v6ll = "fe80::xxx"
if_int_v6   = "2001:xxx"
if_ext_v6   = "2001:xxx"
home_net_v6 = "2001:xxx::/64"

core7       = "10.0.0.xx"
ps3         = "10.0.0.xx"

###############################################################################
# Tables
###############################################################################

table  persist {}
table  persist {}
table  persist {}

table  const persist {   \
    127.0.0.0/8                        \
    10.0.0.0/8                         \
    172.16.0.0/12                      \
    192.168.0.0/16                     \
}

table  const persist {         \
    x.x.x.x                            \
}

table  const persist {        \
    x.x.x.x                            \
}

###############################################################################
# Options
###############################################################################

set skip on lo0
set block-policy return
set loginterface $if_ext1
set state-policy if-bound

###############################################################################
# Packet normalisation
###############################################################################

match on $if_ext1 all scrub (random-id reassemble tcp)
match on $if_ext2 all scrub (random-id reassemble tcp)
match on $if_ext3 all scrub (random-id)

## NOTE: "reassemble tcp" breaks PS3 downloads and may break something else too

###############################################################################
# Translation/redirection rules
###############################################################################

# FTP proxy states need to override the rules below
anchor "ftp-proxy/*"

# NAT
match out on $if_ext3 inet from $ps3 to any nat-to $if_ext3 static-port
match in  on $if_ext3 inet from any to $if_ext3 rdr-to $ps3 rtable 0

match out on $if_ext2 inet from $core7 to any nat-to $if_ext2 static-port
match in  on $if_ext2 inet from any to $if_ext2 rdr-to $core7 rtable 0

match out on $if_ext1 from $home_net_v4 nat-to ($if_ext1)

###############################################################################
# Filter rules
###############################################################################

##
## GENERAL
##

# Block and log everything by default
block log all

# Antispoofing on all interfaces
antispoof quick for $all_ifs

# Block private addresses on external interfaces
block drop in  quick on $ext_ifs from 
block drop out quick on $ext_ifs to   

# Block IPv6 on external IPv4 interfaces
block drop quick on $ext_ifs_v4 inet6 all

# Block IPv4 on external IPv6 interfaces
block drop quick on $ext_ifs_v6 inet all

##
## INCOMING
##

##########
# if_int #
##########

# FTP proxy
pass in quick on $if_int inet proto tcp from $home_net_v4 to port ftp \
    divert-to 127.0.0.1 port 8021

# ps3
pass in quick on $if_int from $ps3 to $if_int_v4
pass in quick on $if_int from $ps3 route-to ($if_ext3 )

# core7
pass in quick on $if_int from $core7 to $if_int_v4
pass in quick on $if_int from $core7 route-to ($if_ext2 )

# Other home network nodes
pass in quick on $if_int from $home_net_v4 to $if_int_v4
pass in quick on $if_int from $home_net_v4 route-to ($if_ext1 )

# IPv6
pass in quick on $if_int from fe80::/16 to $if_int_v6ll
pass in quick on $if_int from fe80::/16 to ff02::/16
pass in quick on $if_int from $home_net_v6

###########
# if_ext1 #
###########

# IPv6 tunneling
pass in quick on $if_ext1 proto icmp from a.b.c.d to ($if_ext1)
pass in quick on $if_ext1 proto ipv6 from x.y.z.w to ($if_ext1)

# Pass in SSH from addresses listed in ssh_ok table
pass in quick on $if_ext1 proto tcp from  to ($if_ext1) port ssh
 
# Pass in HTTP from addresses listed in http_ok table
pass in quick on $if_ext1 proto tcp from  to ($if_ext1) port http
 
###################
# if_ext2 (core7) #
###################

# Steam (https://support.steampowered.com/kb_article.php?ref=8571-GLVN-8711)
pass in quick on $if_ext2 proto tcp to $core7 port 27014:27050
pass in quick on $if_ext2 proto udp to $core7 port 4380
pass in quick on $if_ext2 proto udp to $core7 port 27000:27030

# Black Ops 2
pass in quick on $if_ext2 proto tcp to $core7 port 3074
pass in quick on $if_ext2 proto udp to $core7 port 3074

#################
# if_ext3 (ps3) #
#################

# Nothing

##
## OUTGOING
##

##########
# if_int #
##########

# IPv4 from Internet to home network
pass out quick on $if_int to $ps3 received-on $if_ext3 \
    reply-to ($if_ext3 )
pass out quick on $if_int to $core7 received-on $if_ext2 \
    reply-to ($if_ext2 )
pass out quick on $if_int to $home_net_v4 received-on $if_ext1 \
    reply-to ($if_ext1 )

# IPv4 from fw to home network
pass out quick on $if_int from $if_int_v4 to $home_net_v4

# IPv6 from fw to home network
pass out quick on $if_int from $if_int_v6ll to fe80::/16
pass out quick on $if_int from $if_int_v6ll to ff02::/16
pass out quick on $if_int from $if_int_v6   to $home_net_v6
pass out quick on $if_int from $if_int_v6   to ff02::/16

###########
# if_ext1 #
###########

# IPv6 tunneling
pass out quick on $if_ext1 proto icmp from ($if_ext1) to a.b.c.d
pass out quick on $if_ext1 proto ipv6 from ($if_ext1) to x.y.z.w

# The rest
pass out quick on $if_ext1 inet from ($if_ext1) modulate state

###################
# if_ext2 (core7) #
###################

pass out quick on $if_ext2 inet from ($if_ext2) modulate state rtable 1

#################
# if_ext3 (ps3) #
#################

pass out quick on $if_ext3 inet from ($if_ext3) modulate state rtable 2

############
# if_extv6 #
############

pass out quick on $if_extv6 inet6 from $if_int_v6ll modulate state
pass out quick on $if_extv6 inet6 from $if_ext_v6 modulate state
pass out quick on $if_extv6 inet6 from $home_net_v6 modulate state

In an earlier post, I describe how I have set up my OpenBSD firewall to retrieve multiple public IPs from my ISP (legally – I pay for 5) and map some of those to internal IPs (1:1 NAT or “binat”). One of the downsides of my solution was that I needed one ethernet interface per public IP, as the ISP identifies DHCP clients by their MAC addresses. So, the firewall would need more than 2 ethernet interfaces, which is not an easy requirement for small form-factor firewalls such as Mini-ITX.

One solution to to overcome this with only two ethernet ports in the firewall is to put a managed switch that supports VLAN tagging between the firewall and the xDSL router. I bought an HP Procurve 1810G-8 which does the job perfectly.

My firewall has now only two interfaces, one is “internal”, connected to my home network (unmanaged) switch, and one is “external”, connected to the HP (managed) switch. The external interface is transporting three different VLANs from the firewall to the switch. The VLANs show up just like normal ethernet interfaces on the firewall, so I basically just needed to rename “em1” to “vlanXXX”, “em2” to “vlanYYY”, etc. in a couple of configuration files.

By default, the VLAN interfaces inherit the MAC address of their parent interface. This is not good for my setup, as the DHCP server would see only one client and thus leasing only one IP address. The solution is to use the ifconfig command to set the MAC addresses to unique values. However, the parent interface will drop the received ethernet frame as it does not know anything about the MAC address on the logical VLAN interface. Solution: create a bridge interface and put the parent interface into the bridge. This is the OpenBSD way to put an interface into “promiscuous” mode where it will receive packets with any MAC address, and then this setup will work. I crudely put these bridge creation commands into /etc/rc script before it calls the /etc/netstart, so that the vlan interfaces work when dhclients are started.

A better solution to this “unique-MAC-address-per-VLAN-interface” problem would be to write some new code into OpenBSD network stack so that the parent interface would check its VLAN interfaces’ MAC addresses as well and no promiscuous mode would be needed in this use case. Chances of this being accepted into OpenBSD code base are so slim (nobody knows me and its probably somehow insecure) that I don’t bother to think about any more than this. If you have commit rights and extra time, please go ahead.

Finally, three ports of the switch are connected to the VDSL router, forwarding untagged ethernet frames towards the ISP. From the ISP and DHCP viewpoint, it looks just like three different machines connected to a subscribers modem. And I continue to have my own firewall which does whatever I want it to do.

As I describe in an earlier blog post, I am running an OpenBSD packet filter firewall which has three network interfaces connected to the same ISP. Everything worked so well until the ISP changed something in their configuration and two of the interfaces started to get the same next hop router (gateway) through DHCP configuration. This obviously causes problems with e.g. ARP and routing in general. The solution to this was to start using the “routing domain” feature of OpenBSD.

All interfaces are in routing domain “0” by default. I then set the two “extra” outgoing interfaces to routing domains “1” and “2”. That way each outgoing network interface has its own routing table and ARP table, and routing/ARP problems with the “duplicate next hop” were fixed. However, by definition, routing doesn’t work between routing domains. Luckily I found a way around this by tweaking pf.conf. The solution was to

a) split the “binat” rules and use “rtable” keyword for the rule used for incoming packets, and
b) add the “rtable” keyword for outgoing packets.

That way the route lookup is done on the correct routing tables for both incoming and outgoing packets.

Here are the modified sections in /etc/pf.conf:

# binat on em2 for host "ps3"
match out on $if_ext3 inet from $ps3 to any nat-to $if_ext3 static-port
match in on $if_ext3 inet from any to $if_ext3 rdr-to $ps3 rtable 0

# binat on em1 for host "core7"
match out on $if_ext2 inet from $core7 to any nat-to $if_ext2 static-port
match in on $if_ext2 inet from any to $if_ext2 rdr-to $core7 rtable 0

# NAT on em0 for the rest of the hosts
match out on $if_ext1 from $home_net_v4 nat-to ($if_ext1)

...

pass out quick on $if_ext1 inet from ($if_ext1) modulate state
pass out quick on $if_ext2 inet from ($if_ext2) modulate state rtable 1
pass out quick on $if_ext3 inet from ($if_ext3) modulate state rtable 2