udp: sendmsg ipv4 pktinfo working (macos)

9 files changed, 92 insertions, 53 deletions

diff --git a/.gitignore b/.gitignore
index 5cd5b07..98c6dd2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 target/
 **/*.rs.bk
 .idea/
+.vscode/
diff --git a/Cargo.lock b/Cargo.lock
index 0e7a8c6..a6dedf7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -189,9 +189,9 @@ dependencies = [
  "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools-num 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_derive 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "simplelog 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -430,8 +430,8 @@ dependencies = [
  "pest_derive 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -504,7 +504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 [[package]]
 name = "libc"
 version = "0.2.40"
-source = "git+https://github.com/rust-lang/libc#28b4d1630989318c4a03cacdffd3f8f7651262f4"
+source = "git+https://github.com/rust-lang/libc#69769fbf3bbb9944def0b6b8da56b2cb2a83f2b5"
 
 [[package]]
 name = "libc"
@@ -650,7 +650,7 @@ dependencies = [
 [[package]]
 name = "nix"
 version = "0.11.0-pre"
-source = "git+https://github.com/mcginty/nix?branch=ipv6-pktinfo#7eb2f413dc31ad47e285c196401903d31d9596a6"
+source = "git+https://github.com/mcginty/nix?branch=ipv6-pktinfo#45f2d52f40cef1655d4b6631d9994a964724d449"
 dependencies = [
  "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "cc 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -936,12 +936,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "serde"
-version = "1.0.38"
+version = "1.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "serde_derive"
-version = "1.0.38"
+version = "1.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "proc-macro2 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -961,13 +961,13 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.14"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "itoa 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1016,7 +1016,7 @@ dependencies = [
 
 [[package]]
 name = "socket2"
-version = "0.3.4"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1498,7 +1498,7 @@ dependencies = [
  "pnet_packet 0.21.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "snow 0.1.8-preview (git+https://github.com/mcginty/snow?branch=wireguard)",
- "socket2 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "socket2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "structopt 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "structopt-derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "subtle 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1643,16 +1643,16 @@ dependencies = [
 "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
 "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
-"checksum serde 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)" = "4c36359ac1a823e00db02a243376ced650f088dc1f6259bbf828e4668e3c7399"
-"checksum serde_derive 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)" = "f0477feff739386f5bca8e13fa43d96a4e834904d538f503906c8179f9205f50"
+"checksum serde 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "53e6b6859395f46cf528414659ce43e70902b2277519707c3bd91797b3320330"
+"checksum serde_derive 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "16e97f8dc5b2dabc0183e0cde24b1a53835e5bb3d2c9e0fdb077f895bba7f2a9"
 "checksum serde_derive_internals 0.23.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9d30c4596450fd7bbda79ef15559683f9a79ac0193ea819db90000d7e1cae794"
-"checksum serde_json 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "8f6f1f77b969caa064f347544d703efacaf4854b84831096a5dc206a8aedbc27"
+"checksum serde_json 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7bf1cbb1387028a13739cb018ee0d9b3db534f22ca3c84a5904f7eadfde14e75"
 "checksum simplelog 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ce595117de34b75e057b41e99079e43e9fcc4e5ec9c7ba5f2fea55321f0c624e"
 "checksum slab 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d807fd58c4181bbabed77cb3b891ba9748241a552bcc5be698faaebefc54f46e"
 "checksum slab 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23"
 "checksum slab 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fdeff4cd9ecff59ec7e3744cbca73dfe5ac35c2aedb2cfba8a1c715a18912e9d"
 "checksum snow 0.1.8-preview (git+https://github.com/mcginty/snow?branch=wireguard)" = "<none>"
-"checksum socket2 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "71ebbe82fcdd697244ba7fe6e05e63b5c45910c3927e28469a04947494ff48d8"
+"checksum socket2 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ff606e0486e88f5fc6cfeb3966e434fb409abbc7a3ab495238f70a1ca97f789d"
 "checksum static_slice 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "92a7e0c5e3dfb52e8fbe0e63a1b947bbb17b4036408b151353c4491374931362"
 "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
 "checksum structopt 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "783cb22d520b177a3772e520d04a3c7970d51c3b647ba80739f99be01131b54f"
diff --git a/src/interface/config.rs b/src/interface/config.rs
index 574876d..6d16a84 100644
--- a/src/interface/config.rs
+++ b/src/interface/config.rs
@@ -17,7 +17,6 @@ use std::{cell::RefCell, iter::Iterator, rc::Rc, mem, str};
 use std::fs::{create_dir, remove_file};
 use std::path::{Path, PathBuf};
 use tokio_core::reactor::Handle;
-use udp::Endpoint;
 use types::PeerInfo;
 use hex::FromHex;
 use x25519_dalek as x25519;
diff --git a/src/interface/peer_server.rs b/src/interface/peer_server.rs
index 9f680fd..40a1a23 100644
--- a/src/interface/peer_server.rs
+++ b/src/interface/peer_server.rs
@@ -8,7 +8,6 @@ use time::Timestamp;
 use timer::{Timer, TimerMessage};
 
 use std::convert::TryInto;
-use std::net::SocketAddr;
 use std::time::Duration;
 
 use byteorder::{ByteOrder, LittleEndian};
@@ -77,6 +76,7 @@ impl PeerServer {
             udp.set_mark(fwmark)?;
         }
 
+        // TODO: clear out peer sticky endpoint sources
         self.udp  = Some(udp);
         self.port = Some(port);
         Ok(())
@@ -309,6 +309,7 @@ impl PeerServer {
         match message {
             Rekey(peer_ref, our_index) => {
                 {
+                    // TODO: clear sticky source endpoint if retrying, in case that is the problem
                     let mut peer = peer_ref.borrow_mut();
 
                     match peer.find_session(our_index) {
diff --git a/src/main.rs b/src/main.rs
index 29d447e..0208aaf 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -61,7 +61,7 @@ fn main() {
             ))
         })
         .level(log::LevelFilter::Info)
-        .level_for("wireguard", log::LevelFilter::Debug)
+        .level_for("wireguard", log::LevelFilter::Trace)
         .chain(std::io::stdout())
         .apply().unwrap();
 
diff --git a/src/peer.rs b/src/peer.rs
index a7ee5f7..4899893 100644
--- a/src/peer.rs
+++ b/src/peer.rs
@@ -12,7 +12,6 @@ use message::{Initiation, Response, CookieReply, Transport};
 use std::{self, mem};
 use std::collections::VecDeque;
 use std::fmt::{self, Debug, Display, Formatter};
-use std::net::SocketAddr;
 use std::time::{SystemTime, UNIX_EPOCH};
 use hex;
 use time::{Tai64n, Timestamp};
diff --git a/src/types.rs b/src/types.rs
index cc65105..3ed6f64 100644
--- a/src/types.rs
+++ b/src/types.rs
@@ -1,6 +1,6 @@
 use base64;
 use std::fmt::{self, Display, Formatter};
-use std::net::{IpAddr, SocketAddr};
+use std::net::IpAddr;
 use udp::Endpoint;
 
 #[derive(Clone, Debug, Default)]
diff --git a/src/udp/frame.rs b/src/udp/frame.rs
index 239a58d..b0f1dd4 100644
--- a/src/udp/frame.rs
+++ b/src/udp/frame.rs
@@ -29,6 +29,7 @@ impl Stream for UdpFramed {
     type Error = io::Error;
 
     fn poll(&mut self) -> Poll<Option<PeerServerMessage>, io::Error> {
+        
         let (n, addr) = try_nb!(self.socket.recv_from(&mut self.rd));
         trace!("received {} bytes, decoding", n);
         let frame = self.codec.decode(&addr, &self.rd[..n])?;
diff --git a/src/udp/mod.rs b/src/udp/mod.rs
index b6bca9e..1c79750 100644
--- a/src/udp/mod.rs
+++ b/src/udp/mod.rs
@@ -14,10 +14,12 @@ use nix::sys::{uio::IoVec,
                    in_pktinfo,
                    CmsgSpace,
                    ControlMessage,
+                   InetAddr,
                    UnknownCmsg,
                    MsgFlags,
                    SockAddr,
-                   recvmsg
+                   recvmsg,
+                   sendmsg
                }};
 use socket2::{Socket, Domain, Type, Protocol};
 
@@ -34,29 +36,46 @@ pub struct UdpSocket {
     handle: Handle,
 }
 
+// I understand that, ex., the V4 enum should really hold a SocketAddrV4 struct,
+// but this is for simplicity because nix only offers a to_std() that returns
+// `SocketAddr` from its `SockAddr`, so it makes the code cleaner with little
+// performance impact.
 #[derive(Clone, Copy, Debug)]
-pub struct Endpoint {
-    pub addr: SocketAddr,
-    pub pktinfo: Option<PktInfo>,
+pub enum Endpoint {
+    V4(SocketAddr, Option<in_pktinfo>),
+    V6(SocketAddr, Option<in6_pktinfo>)
+}
+
+impl Endpoint {
+    fn addr(&self) -> SocketAddr {
+        match *self {
+            Endpoint::V4(sock, _) => sock,
+            Endpoint::V6(sock, _) => sock,
+        }
+    }
 }
 
 impl Deref for Endpoint {
     type Target = SocketAddr;
 
     fn deref(&self) -> &<Self as Deref>::Target {
-        &self.addr
+        match *self {
+            Endpoint::V4(ref sock, _) => sock,
+            Endpoint::V6(ref sock, _) => sock
+        }
     }
 }
 
 impl From<SocketAddr> for Endpoint {
     fn from(addr: SocketAddr) -> Self {
-        Endpoint {
-            addr,
-            pktinfo: None
+        match addr {
+            SocketAddr::V4(_) => Endpoint::V4(addr, None),
+            SocketAddr::V6(_) => Endpoint::V6(addr, None),
         }
     }
 }
 
+// TODO: support linux
 /// IPV6_RECVPKTINFO is missing from the libc crate. Value taken from https://git.io/vxNel.
 pub const IPV6_RECVPKTINFO : i32 = 61;
 pub const IP_PKTINFO       : i32 = 26;
@@ -81,9 +100,8 @@ impl UdpSocket {
                                        mem::size_of_val(&on) as libc::socklen_t);
             if ret != 0 {
                 let err: Result<(), _> = Err(io::Error::last_os_error());
-                err.expect("setsockopt failed");
+                err.expect("setsockopt IP_PKTINFO failed");
             }
-            debug!("set IP_PKTINFO");
         }
 
         unsafe {
@@ -94,10 +112,8 @@ impl UdpSocket {
                                        mem::size_of_val(&on) as libc::socklen_t);
             if ret != 0 {
                 let err: Result<(), _> = Err(io::Error::last_os_error());
-                err.expect("setsockopt failed");
+                err.expect("setsockopt IPV6_RECVPKTINFO failed");
             }
-
-            debug!("set IPV6_PKTINFO");
         }
 
         socket4.set_nonblocking(true)?;
@@ -127,10 +143,10 @@ impl UdpSocket {
         Ok((self.io4.get_ref().local_addr()?, self.io6.get_ref().local_addr()?))
     }
 
-    fn get_io(&self, addr: &SocketAddr) -> &PollEvented<mio::net::UdpSocket> {
-        match *addr {
-            SocketAddr::V4(_) => &self.io4,
-            SocketAddr::V6(_) => &self.io6,
+    fn get_io(&self, endpoint: &Endpoint) -> &PollEvented<mio::net::UdpSocket> {
+        match *endpoint {
+            Endpoint::V4(..) => &self.io4,
+            Endpoint::V6(..) => &self.io6,
         }
     }
 
@@ -171,13 +187,35 @@ impl UdpSocket {
             return Err(io::ErrorKind::WouldBlock.into())
         }
 
-        match io.get_ref().send_to(buf, target) {
-            Ok(n) => Ok(n),
+        let cmsgs = match *target {
+            Endpoint::V4(addr, Some(ref pktinfo)) => vec![ControlMessage::Ipv4PacketInfo(pktinfo)],
+            Endpoint::V6(addr, Some(ref pktinfo)) => vec![ControlMessage::Ipv6PacketInfo(pktinfo)],
+            _                                     => vec![]
+        };
+
+        match *target {
+            Endpoint::V4(addr, Some(ref pktinfo)) => trace!("sending cmsg: {:?}", pktinfo),
+            Endpoint::V6(addr, Some(ref pktinfo)) => trace!("sending cmsg: {:?}", pktinfo),
+            _                                     => trace!("not sending any pktinfo")
+        }
+
+        let res = sendmsg(io.get_ref().as_raw_fd(),
+                          &[IoVec::from_slice(buf)],
+                          &cmsgs,
+                          MsgFlags::empty(),
+                          Some(&SockAddr::Inet(InetAddr::from_std(&target.addr()))));
+
+        match res {
+            Ok(len) => Ok(len),
+            Err(nix::Error::Sys(Errno::EAGAIN)) => {
+                io.need_write();
+                Err(io::ErrorKind::WouldBlock.into())
+            },
+            Err(nix::Error::Sys(errno)) => {
+                Err(io::Error::last_os_error())
+            },
             Err(e) => {
-                if e.kind() == io::ErrorKind::WouldBlock {
-                    io.need_write();
-                }
-                Err(e)
+                Err(io::Error::new(io::ErrorKind::Other, e))
             }
         }
     }
@@ -206,19 +244,19 @@ impl UdpSocket {
                                    Ipv4Addr::from(info.ipi_addr),
                                    Ipv4Addr::from(info.ipi_spec_dst),
                                    info.ipi_ifindex);
-                            Ok((msg.bytes, Endpoint {
-                                addr: addr.to_std(),
-                                pktinfo: Some(PktInfo::V4(info.clone()))
-                            }))
+                            let endpoint = Endpoint::V4(addr.to_std(), Some(in_pktinfo {
+                                ipi_addr    : [0u8; 4],
+                                ipi_spec_dst: info.ipi_addr,
+                                ipi_ifindex : info.ipi_ifindex,
+                            }));
+                            Ok((msg.bytes, endpoint))
                         },
                         Some(ControlMessage::Ipv6PacketInfo(info)) => {
                             trace!("ipv6 cmsg (\n  ipi6_addr: {:?},\n  ipi6_ifindex: {}\n)",
                                    Ipv6Addr::from(info.ipi6_addr),
                                    info.ipi6_ifindex);
-                            Ok((msg.bytes, Endpoint {
-                                addr: addr.to_std(),
-                                pktinfo: Some(PktInfo::V6(info.clone()))
-                            }))
+                            let endpoint = Endpoint::V6(addr.to_std(), Some(*info));
+                            Ok((msg.bytes, endpoint))
                         },
                         _ => Err(io::Error::new(io::ErrorKind::Other, "missing pktinfo"))
                     }