//! Parsing implementation for Tor microdescriptors. //! //! A "microdescriptor" is an incomplete, infrequently-changing //! summary of a relay's information that is generated by //! the directory authorities. //! //! Microdescriptors are much smaller than router descriptors, and //! change less frequently. For this reason, they're currently used //! for building circuits by all relays and clients. //! //! Microdescriptors can't be used on their own: you need to know //! which relay they are for, which requires a valid consensus //! directory. use crate::parse::keyword::Keyword; use crate::parse::parser::SectionRules; use crate::parse::tokenize::{ItemResult, NetDocReader}; use crate::types::family::{RelayFamily, RelayFamilyId, RelayFamilyIds}; use crate::types::misc::*; use crate::types::policy::PortPolicy; use crate::util; use crate::util::PeekableIterator; use crate::util::str::Extent; use crate::{AllowAnnotations, Error, NetdocErrorKind as EK, Result}; use tor_basic_utils::intern::Intern; use tor_error::internal; use tor_llcrypto::d; use tor_llcrypto::pk::{curve25519, ed25519, rsa}; use derive_deftly::Deftly; use digest::Digest; use std::str::FromStr as _; use std::sync::LazyLock; use std::time; #[cfg(feature = "build_docs")] mod build; #[cfg(feature = "build_docs")] pub use build::MicrodescBuilder; /// Length of a router microdescriptor digest pub const DOC_DIGEST_LEN: usize = 32; /// Annotations prepended to a microdescriptor that has been stored to /// disk. #[allow(dead_code)] #[derive(Clone, Debug, Default)] pub struct MicrodescAnnotation { /// A time at which this microdescriptor was last listed in some /// consensus document. last_listed: Option, } /// The digest of a microdescriptor as used in microdesc consensuses pub type MdDigest = [u8; DOC_DIGEST_LEN]; /// A single microdescriptor. /// /// #[derive(Clone, Debug, Deftly, PartialEq, Eq)] #[derive_deftly(Constructor, NetdocEncodable, NetdocParseable)] #[allow(clippy::exhaustive_structs)] pub struct Microdesc { /// The legacy onion key, whose object is optional but whose item serves /// as the intro line for these kind of descriptors. pub onion_key: MicrodescIntroItem, /// Public key used for the ntor circuit extension protocol. #[deftly(constructor)] #[deftly(netdoc(single_arg))] pub ntor_onion_key: Curve25519Public, /// Declared family for this relay. #[deftly(netdoc(default(skip)))] pub family: Intern, /// Family identities for this relay. #[deftly(netdoc(default(skip)))] pub family_ids: RelayFamilyIds, /// List of IPv4 ports to which this relay will exit #[deftly(netdoc(keyword = "p", default(skip)))] pub ipv4_policy: Intern, /// List of IPv6 ports to which this relay will exit #[deftly(netdoc(keyword = "p6", default(skip)))] pub ipv6_policy: Intern, /// Ed25519 identity for this relay // TODO SPEC: Set this to "exactly once". #[deftly(constructor)] #[deftly(netdoc(keyword = "id", with = "Ed25519IdentityLine"))] pub ed25519_id: Ed25519IdentityLine, // addr is obsolete and doesn't go here any more // pr is obsolete and doesn't go here any more. #[doc(hidden)] #[deftly(netdoc(skip))] pub __non_exhaustive: (), } /// A single microdescriptor and also its SHA256 hash /// /// API compatibility type. /// /// This type is only generated when the microdescriptor is parsed /// using the old parser ([`MicrodescAndHash::parse`]) /// rather than the new one /// (`Microdesc as `[`NetdocParseable`](crate::parse2::NetdocParseable)). #[derive(Clone, Debug, Deftly, PartialEq, Eq, derive_more::Deref, derive_more::DerefMut)] #[non_exhaustive] pub struct MicrodescAndHash { /// The microdescriptor #[deref] #[deref_mut] pub md: Microdesc, /// The SHA256 digest of the text of this microdescriptor. /// /// This value is used to identify the microdescriptor when /// downloading it, and when listing it in a consensus document. pub sha256: MdDigest, } impl Microdesc { /// Return the ntor onion key for this microdesc pub fn ntor_key(&self) -> &curve25519::PublicKey { &self.ntor_onion_key.0 } /// Return the ipv4 exit policy for this microdesc pub fn ipv4_policy(&self) -> &Intern { &self.ipv4_policy } /// Return the ipv6 exit policy for this microdesc pub fn ipv6_policy(&self) -> &Intern { &self.ipv6_policy } /// Return the relay family for this microdesc pub fn family(&self) -> &RelayFamily { self.family.as_ref() } /// Return the ed25519 identity for this microdesc, if its /// Ed25519 identity is well-formed. pub fn ed25519_id(&self) -> &ed25519::Ed25519Identity { &self.ed25519_id.pk.0 } /// Return a list of family ids for this microdesc. pub fn family_ids(&self) -> &[RelayFamilyId] { self.family_ids.as_ref() } } impl MicrodescAndHash { /// Create a new MicrodescBuilder that can be used to construct /// microdescriptors. /// /// This function is only available when the crate is built with the /// `build_docs` feature. /// /// # Limitations /// /// The generated microdescriptors cannot yet be encoded, and do /// not yet have correct sha256 digests. As such they are only /// useful for testing. #[cfg(feature = "build_docs")] pub fn builder() -> MicrodescBuilder { MicrodescBuilder::new() } /// Return the sha256 digest of this microdesc. pub fn digest(&self) -> &MdDigest { &self.sha256 } } /// Intro line for a [`Microdesc`]. /// /// The object (the onion key) is deprecated and optional, but the item itself /// must be present, because it is used to mark the start of the netdoc. /// /// The object is private to prevent interfacing applications /// from generating microdesc's with an onion-key; they are not necessary /// anymore and just waste space. #[derive(Debug, Clone, Default, Deftly, PartialEq, Eq)] #[derive_deftly(ItemValueEncodable, ItemValueParseable)] pub struct MicrodescIntroItem(#[deftly(netdoc(object))] Option); /// A microdescriptor annotated with additional data /// /// TODO: rename this. #[allow(dead_code)] #[derive(Clone, Debug)] pub struct AnnotatedMicrodesc { /// The microdescriptor md: MicrodescAndHash, /// The annotations for the microdescriptor ann: MicrodescAnnotation, /// Where did we find the microdescriptor with the originally parsed /// string? location: Option, } impl AnnotatedMicrodesc { /// Consume this annotated microdesc and discard its annotations. pub fn into_microdesc(self) -> MicrodescAndHash { self.md } /// Return a reference to the microdescriptor within this annotated /// microdescriptor. pub fn md(&self) -> &MicrodescAndHash { &self.md } /// If this Microdesc was parsed from `s`, return its original text. pub fn within<'a>(&self, s: &'a str) -> Option<&'a str> { self.location.as_ref().and_then(|ext| ext.reconstruct(s)) } } decl_keyword! { /// Keyword type for recognized objects in microdescriptors. MicrodescKwd { annotation "@last-listed" => ANN_LAST_LISTED, "onion-key" => ONION_KEY, "ntor-onion-key" => NTOR_ONION_KEY, "family" => FAMILY, "family-ids" => FAMILY_IDS, "p" => P, "p6" => P6, "id" => ID, } } /// Rules about annotations that can appear before a Microdescriptor static MICRODESC_ANNOTATIONS: LazyLock> = LazyLock::new(|| { use MicrodescKwd::*; let mut rules = SectionRules::builder(); rules.add(ANN_LAST_LISTED.rule().args(1..)); rules.add(ANN_UNRECOGNIZED.rule().may_repeat().obj_optional()); // unrecognized annotations are okay; anything else is a bug in this // context. rules.reject_unrecognized(); rules.build() }); /// Rules about entries that must appear in an Microdesc, and how they must /// be formed. static MICRODESC_RULES: LazyLock> = LazyLock::new(|| { use MicrodescKwd::*; let mut rules = SectionRules::builder(); rules.add(ONION_KEY.rule().required().no_args().obj_optional()); rules.add(NTOR_ONION_KEY.rule().required().args(1..)); rules.add(FAMILY.rule().args(1..)); rules.add(FAMILY_IDS.rule().args(0..)); rules.add(P.rule().args(2..)); rules.add(P6.rule().args(2..)); rules.add(ID.rule().may_repeat().args(2..)); rules.add(UNRECOGNIZED.rule().may_repeat().obj_optional()); rules.build() }); impl MicrodescAnnotation { /// Extract a (possibly empty) microdescriptor annotation from a /// reader. #[allow(dead_code)] fn parse_from_reader( reader: &mut NetDocReader<'_, MicrodescKwd>, ) -> Result { use MicrodescKwd::*; let mut items = reader.pause_at(|item| item.is_ok_with_non_annotation()); let body = MICRODESC_ANNOTATIONS.parse(&mut items)?; let last_listed = match body.get(ANN_LAST_LISTED) { None => None, Some(item) => Some(item.args_as_str().parse::()?.into()), }; Ok(MicrodescAnnotation { last_listed }) } } impl MicrodescAndHash { /// Parse a string into a new microdescriptor. pub fn parse(s: &str) -> Result { let mut items = crate::parse::tokenize::NetDocReader::new(s)?; let (result, _) = Self::parse_from_reader(&mut items).map_err(|e| e.within(s))?; items.should_be_exhausted()?; Ok(result) } /// Extract a single microdescriptor from a NetDocReader. fn parse_from_reader( reader: &mut NetDocReader<'_, MicrodescKwd>, ) -> Result<(MicrodescAndHash, Option)> { use MicrodescKwd::*; let s = reader.str(); let mut first_onion_key = true; // We'll pause at the next annotation, or at the _second_ onion key. let mut items = reader.pause_at(|item| match item { Err(_) => false, Ok(item) => { item.kwd().is_annotation() || if item.kwd() == ONION_KEY { let was_first = first_onion_key; first_onion_key = false; !was_first } else { false } } }); let body = MICRODESC_RULES.parse(&mut items)?; // We have to start with onion-key let start_pos = { // unwrap here is safe because parsing would have failed // had there not been at least one item. #[allow(clippy::unwrap_used)] let first = body.first_item().unwrap(); if first.kwd() != ONION_KEY { return Err(EK::WrongStartingToken .with_msg(first.kwd_str().to_string()) .at_pos(first.pos())); } // Unwrap is safe here because we are parsing these strings from s #[allow(clippy::unwrap_used)] util::str::str_offset(s, first.kwd_str()).unwrap() }; // Legacy (tap) onion key. We parse this to make sure it's well-formed, // but then we discard it immediately, since we never want to use it. // // In microdescriptors, the ONION_KEY field is mandatory, but its // associated object is optional. { let tok = body.required(ONION_KEY)?; if tok.has_obj() { let _: rsa::PublicKey = tok .parse_obj::("RSA PUBLIC KEY")? .check_len_eq(1024)? .check_exponent(65537)? .into(); } } // Ntor onion key let ntor_onion_key = body .required(NTOR_ONION_KEY)? .parse_arg::(0)?; // family // // (We don't need to add the relay's own ID to this family, as we do in // RouterDescs: the authorities already took care of that for us.) let family = body .maybe(FAMILY) .parse_args_as_str::()? .unwrap_or_else(RelayFamily::new) .intern(); // Family ids (happy families case). let family_ids = body .maybe(FAMILY_IDS) .args_as_str() .unwrap_or("") .split_ascii_whitespace() .map(RelayFamilyId::from_str) .collect::>()?; // exit policies. let ipv4_policy = body .maybe(P) .parse_args_as_str::()? .unwrap_or_else(PortPolicy::new_reject_all); let ipv6_policy = body .maybe(P6) .parse_args_as_str::()? .unwrap_or_else(PortPolicy::new_reject_all); // ed25519 identity let ed25519_id = { let id_tok = body .slice(ID) .iter() .find(|item| item.arg(0) == Some("ed25519")); match id_tok { None => { return Err(EK::MissingToken.with_msg("id ed25519")); } Some(tok) => Ed25519IdentityLine { alg: Ed25519AlgorithmString::Ed25519, pk: tok.parse_arg::(1)?, }, } }; let end_pos = { // unwrap here is safe because parsing would have failed // had there not been at least one item. #[allow(clippy::unwrap_used)] let last_item = body.last_item().unwrap(); last_item.offset_after(s).ok_or_else(|| { Error::from(internal!("last item was not within source string")) .at_pos(last_item.end_pos()) })? }; let text = s.get(start_pos..end_pos).ok_or(internal!("chopped utf8"))?; let sha256 = d::Sha256::digest(text.as_bytes()).into(); let location = Extent::new(s, text); let md = Microdesc { onion_key: Default::default(), ntor_onion_key, family, ipv4_policy: ipv4_policy.intern(), ipv6_policy: ipv6_policy.intern(), ed25519_id, family_ids, __non_exhaustive: (), }; let md = MicrodescAndHash { md, sha256 }; Ok((md, location)) } } /// Consume tokens from 'reader' until the next token is the beginning /// of a microdescriptor: an annotation or an ONION_KEY. If no such /// token exists, advance to the end of the reader. fn advance_to_next_microdesc(reader: &mut NetDocReader<'_, MicrodescKwd>, annotated: bool) { use MicrodescKwd::*; loop { let item = reader.peek(); match item { Some(Ok(t)) => { let kwd = t.kwd(); if (annotated && kwd.is_annotation()) || kwd == ONION_KEY { return; } } Some(Err(_)) => { // We skip over broken tokens here. // // (This case can't happen in practice, since if there had been // any error tokens, they would have been handled as part of // handling the previous microdesc.) } None => { return; } }; let _ = reader.next(); } } /// An iterator that parses one or more (possibly annotated) /// microdescriptors from a string. #[derive(Debug)] pub struct MicrodescReader<'a> { /// True if we accept annotations; false otherwise. annotated: bool, /// An underlying reader to give us Items for the microdescriptors reader: NetDocReader<'a, MicrodescKwd>, } impl<'a> MicrodescReader<'a> { /// Construct a MicrodescReader to take microdescriptors from a string /// 's'. pub fn new(s: &'a str, allow: &AllowAnnotations) -> Result { let reader = NetDocReader::new(s)?; let annotated = allow == &AllowAnnotations::AnnotationsAllowed; Ok(MicrodescReader { annotated, reader }) } /// If we're annotated, parse an annotation from the reader. Otherwise /// return a default annotation. fn take_annotation(&mut self) -> Result { if self.annotated { MicrodescAnnotation::parse_from_reader(&mut self.reader) } else { Ok(MicrodescAnnotation::default()) } } /// Parse a (possibly annotated) microdescriptor from the reader. /// /// On error, parsing stops after the first failure. fn take_annotated_microdesc_raw(&mut self) -> Result { let ann = self.take_annotation()?; let (md, location) = MicrodescAndHash::parse_from_reader(&mut self.reader)?; Ok(AnnotatedMicrodesc { md, ann, location }) } /// Parse a (possibly annotated) microdescriptor from the reader. /// /// On error, advance the reader to the start of the next microdescriptor. fn take_annotated_microdesc(&mut self) -> Result { let pos_orig = self.reader.pos(); let result = self.take_annotated_microdesc_raw(); if result.is_err() { if self.reader.pos() == pos_orig { // No tokens were consumed from the reader. We need to // drop at least one token to ensure we aren't looping. // // (This might not be able to happen, but it's easier to // explicitly catch this case than it is to prove that // it's impossible.) let _ = self.reader.next(); } advance_to_next_microdesc(&mut self.reader, self.annotated); } result } } impl<'a> Iterator for MicrodescReader<'a> { type Item = Result; fn next(&mut self) -> Option { // If there is no next token, we're at the end. self.reader.peek()?; Some( self.take_annotated_microdesc() .map_err(|e| e.within(self.reader.str())), ) } } #[cfg(test)] mod test { // @@ begin test lint list maintained by maint/add_warning @@ #![allow(clippy::bool_assert_comparison)] #![allow(clippy::clone_on_copy)] #![allow(clippy::dbg_macro)] #![allow(clippy::mixed_attributes_style)] #![allow(clippy::print_stderr)] #![allow(clippy::print_stdout)] #![allow(clippy::single_char_pattern)] #![allow(clippy::unwrap_used)] #![allow(clippy::unchecked_time_subtraction)] #![allow(clippy::useless_vec)] #![allow(clippy::needless_pass_by_value)] #![allow(clippy::string_slice)] // See arti#2571 //! use super::*; use crate::encode::encode_netdoc_unsigned; use hex_literal::hex; const TESTDATA: &str = include_str!("../../testdata/microdesc1.txt"); const TESTDATA2: &str = include_str!("../../testdata/microdesc2.txt"); const TESTDATA3: &str = include_str!("../../testdata/microdesc3.txt"); const TESTDATA4: &str = include_str!("../../testdata/microdesc4.txt"); fn read_bad(fname: &str) -> String { use std::fs; use std::path::PathBuf; let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push("testdata"); path.push("bad-mds"); path.push(fname); fs::read_to_string(path).unwrap() } #[test] fn parse_single() -> Result<()> { let _md = MicrodescAndHash::parse(TESTDATA)?; Ok(()) } #[test] fn parse_no_tap_key() -> Result<()> { let _md = MicrodescAndHash::parse(TESTDATA3)?; Ok(()) } #[test] fn parse_multi() -> Result<()> { use humantime::parse_rfc3339; let mds: Result> = MicrodescReader::new(TESTDATA2, &AllowAnnotations::AnnotationsAllowed)?.collect(); let mds = mds?; assert_eq!(mds.len(), 4); assert_eq!( mds[0].ann.last_listed.unwrap(), parse_rfc3339("2020-01-27T18:52:09Z").unwrap() ); assert_eq!( mds[0].md().digest(), &hex!("38c71329a87098cb341c46c9c62bd646622b4445f7eb985a0e6adb23a22ccf4f") ); assert_eq!( mds[0].md().ntor_key().as_bytes(), &hex!("5e895d65304a3a1894616660143f7af5757fe08bc18045c7855ee8debb9e6c47") ); assert!(mds[0].md().ipv4_policy().allows_port(993)); assert!(mds[0].md().ipv6_policy().allows_port(993)); assert!(!mds[0].md().ipv4_policy().allows_port(25)); assert!(!mds[0].md().ipv6_policy().allows_port(25)); assert_eq!( mds[0].md().ed25519_id().as_bytes(), &hex!("2d85fdc88e6c1bcfb46897fca1dba6d1354f93261d68a79e0b5bc170dd923084") ); Ok(()) } #[test] fn parse_family_ids() -> Result<()> { let mds: Vec = MicrodescReader::new(TESTDATA4, &AllowAnnotations::AnnotationsNotAllowed)? .collect::>()?; assert_eq!(mds.len(), 2); let md0 = mds[0].md(); let md1 = mds[1].md(); assert!(md0.family_ids().is_empty()); assert_eq!( md1.family_ids(), &[ "ed25519:dXMgdGhlIHRyaXVtcGguICAgIC1UaG9tYXMgUGFpbmU" .parse() .unwrap(), "other:Example".parse().unwrap() ] ); assert!(matches!(md1.family_ids()[0], RelayFamilyId::Ed25519(_))); Ok(()) } #[test] fn test_bad() { use crate::Pos; use crate::types::policy::PolicyError; fn check(fname: &str, e: &Error) { let content = read_bad(fname); let res = MicrodescAndHash::parse(&content); assert!(res.is_err()); assert_eq!(&res.err().unwrap(), e); } check( "wrong-start", &EK::WrongStartingToken .with_msg("family") .at_pos(Pos::from_line(1, 1)), ); check( "bogus-policy", &EK::BadPolicy .at_pos(Pos::from_line(9, 1)) .with_source(PolicyError::InvalidPort), ); check( "non-ascii-policy", &EK::BadPolicy .at_pos(Pos::from_line(9, 1)) .with_source(PolicyError::InvalidPort), ); check("wrong-id", &EK::MissingToken.with_msg("id ed25519")); } #[test] fn test_recover() -> Result<()> { let mut data = read_bad("wrong-start"); data += TESTDATA; data += &read_bad("wrong-id"); let res: Vec> = MicrodescReader::new(&data, &AllowAnnotations::AnnotationsAllowed)?.collect(); assert_eq!(res.len(), 3); assert!(res[0].is_err()); assert!(res[1].is_ok()); assert!(res[2].is_err()); Ok(()) } /// Checks whether parse2 works on [`Microdesc`]. /// /// Certain values such as public keys are hardcoded and can be simply /// replaced by a copy and paste in the case one replaces the testdata2 /// vector's in the future. #[test] fn parse2() -> anyhow::Result<()> { use tor_llcrypto::pk::ed25519::Ed25519Identity; use crate::parse2; let md = include_str!("../../testdata2/cached-microdescs.new"); let mds = parse2::parse_netdoc_multiple::(&parse2::ParseInput::new( md, "../../testdata2/cached-microdescs.new", )) .unwrap(); assert_eq!(mds.len(), 7); assert_eq!( mds[0], Microdesc { onion_key: MicrodescIntroItem(rsa::PublicKey::from_der( pem::parse( " -----BEGIN RSA PUBLIC KEY----- MIGJAoGBANF8Zgxp8amY1esYdPj2Ada1ORiVB/A4sgKLQ5ij/wsasO3yjjLcvHRB UJ0mAQWql/nauvjnKUeZFcGm3t7q0v3F9uUsOGTAZ/IKh31UQAm5OS/TJyf8IHky Yl0wCKpUZFHs5CHsajLSfXZKHkwfqRXFEJu9aMtmQdQFfqE9JOJHAgMBAAE= -----END RSA PUBLIC KEY----- " ) .unwrap() .contents() )), ntor_onion_key: curve25519::PublicKey::from(<[u8; 32]>::from( FixedB64::<32>::from_str("I1S8JfcqPPHWVTxfjq/eGmGiu/OtR+fF0Z86Ge1mq3s") .unwrap() )) .into(), family: Default::default(), ipv4_policy: Default::default(), ipv6_policy: Default::default(), ed25519_id: Ed25519Identity::from(<[u8; 32]>::from( FixedB64::<32>::from_str("yhO6nETO5AUdvJbLgPnw4mFjozGXWMCqOp30nY6nM8E") .unwrap() )) .into(), family_ids: Default::default(), __non_exhaustive: (), } ); let enc = encode_netdoc_unsigned(&mds)?; let exp = md; assert_eq_or_diff!(&enc, &exp); Ok(()) } /// Manual test for happy families. // TODO: This should be included in testdata2/ but that would require the // chutney/shadow integration test to actually do families at all. #[test] fn parse2_happy_family() { use tor_llcrypto::pk::ed25519::Ed25519Identity; use crate::parse2::{self, ParseInput}; use std::iter; // A microdescriptor taken from the wild containing happy families. const MICRODESC: &str = "\ onion-key -----BEGIN RSA PUBLIC KEY----- MIGJAoGBAMk57F7qGHVadBJ6m4028w13I1Qk67Ee0JU88w7NObKBph3DQYjgYs4e eUdiW4Gdsx8w/xOuK0foCo0O8Iqq5MXtVcpUP/N+5uB7SVvGdJFsKw21KdIc6v8g ACZAijw5ZPOdhLbyLQyFHNV8zXUov1dlx/Fb9M3lPMVevnDbuKM5AgMBAAE= -----END RSA PUBLIC KEY----- ntor-onion-key fhhP23UKD4L2jehA5gopAo5b6NSoB+kZN5Q4ULv3Zww family $4CFFD403DAB89A689F3FDB80B5366E46D879E736 $4D6C1486939A42D7FFE69BCD9F3FDAA86C743433 $73955E6A69BA5E0827F48206CAD78C045BBE8873 $8DBA9ADCA5B3A3AB6D2B4F88AC2F96614D33DAB3 $B29E3E30443F897F48B86765F1BC1DB917F5DF46 $CD642E7E722979580B6D631697772C0B72BCF25C $D9E7B6A73C8278274081B77D373ECCE4552E75FB $F2515315FE0DB7456194CABC503B526B49951415 family-ids ed25519:b54cKgML0ykRyhdIRcq1xtW19iEVsMYnGNbdY+vvcas id ed25519 /MU/FVKRGcZAy8XFnzLS6Dgcg6s1VpYeFjkwb6+CVhw "; let md = parse2::parse_netdoc::(&ParseInput::new(MICRODESC, "")).unwrap(); assert_eq!( md.family_ids, RelayFamilyIds::from_iter(iter::once(RelayFamilyId::Ed25519( Ed25519Identity::from_base64("b54cKgML0ykRyhdIRcq1xtW19iEVsMYnGNbdY+vvcas") .unwrap() ))) ); } }