Skip to main content

dayzmake_core/
preprocess.rs

1//! Functions for preprocessing Arma configs and scripts
2
3use std::cell::Cell;
4use std::clone::Clone;
5use std::collections::HashMap;
6use std::env::current_dir;
7use std::ffi::OsStr;
8use std::fs::{File, read_dir};
9use std::io::{Error, Read};
10use std::path::{Component, Path, PathBuf};
11
12use crate::error::{ErrorExt, PreprocessParseErrorExt, printable};
13use crate::warning::{Warning, WarningSink};
14
15peg::parser! {
16    /// PEG grammar for the preprocessor: directives, macros, comments and tokens. `deep` is set when
17    /// macro argument parentheses nest deeper than [`MAX_NESTING_DEPTH`]; the caller turns that into an error.
18    pub(crate) grammar preprocess_grammar(deep: &Cell<bool>, file_end: bool) for str {
19        rule newline() = "\r\n" / "\n"
20
21        rule name() -> String = n:$(['a'..='z' | 'A'..='Z' | '0'..='9' | '_']+) {
22            n.to_string()
23        }
24
25        rule include_path() -> String =
26            "\"" path:$((!['"'] [_])*) "\"" { path.to_string() } /
27            "<" path:$((!['>'] [_])*) ">"   { path.to_string() }
28
29        rule parameters() -> Vec<String> = "(" [' ' | '\t']* p:(name() ** ([' ' | '\t']* "," [' ' | '\t']*)) [' ' | '\t']* ")" {
30            p
31        }
32
33        rule definition_value() -> Vec<Token> = ([' ' | '\t']+ / &("\\" newline())) v:(token()*) {
34            v
35        }
36
37        rule definition() -> Definition = n:name() p:parameters()? v:definition_value()? {
38            Definition {
39                name: n,
40                parameters: p,
41                value: v.unwrap_or_default(),
42                local: false
43            }
44        }
45
46        rule directive() -> Directive =
47            "#" [' ' | '\t']* "include" [' ' | '\t']+ path:include_path() { Directive::IncludeDirective(path) } /
48            "#" [' ' | '\t']* "define" [' ' | '\t']+ d:definition() { Directive::DefineDirective(d) } /
49            "#" [' ' | '\t']* "undef" [' ' | '\t']+ n:name() { Directive::UndefDirective(n) } /
50            "#" [' ' | '\t']* "ifdef" [' ' | '\t']+ n:name() { Directive::IfDefDirective(n) } /
51            "#" [' ' | '\t']* "ifndef" [' ' | '\t']+ n:name() { Directive::IfNDefDirective(n) } /
52            "#" [' ' | '\t']* "else" { Directive::ElseDirective } /
53            "#" [' ' | '\t']* "endif" { Directive::EndIfDirective }
54
55        rule nesting_ok(d: usize) = {?
56            if deep.get() || d >= MAX_NESTING_DEPTH {
57                deep.set(true);
58                Err("parentheses nested within the limit")
59            } else {
60                Ok(())
61            }
62        }
63
64        rule arg_rec(d: usize) = "(" nesting_ok(d) (arg_rec(d + 1) / "\\\\" / ("\\" newline()) / !['\r' | '\n' | '(' | ')'] [_])* ")"
65
66        rule argument() -> String = a:$((arg_rec(0) / "\\\\" / ("\\" newline()) / !['\r' | '\n' | ',' | ')'] [_])*) {
67            a.to_string()
68        }
69
70        /// Parses a macro argument list: `(a, b, c)`.
71        pub rule arguments() -> Vec<String> = [' ' | '\t']* "(" [' ' | '\t']* a:(argument() ** ([' ' | '\t']* "," [' ' | '\t']*)) [' ' | '\t']* ")" {
72            a
73        }
74
75        rule nonmacro_token() -> String = s:$((!macro_proper() !comment_token() !concat_token() ("\\\\" / ("\\" newline()) / !['"' | '\r' | '\n'] [_]))+) {
76            s.to_string()
77        }
78
79        rule string_token() -> (String, u32) =
80                s:$("\"" ("\\\\" / ("\\" newline()) / "\"\"" / !['\r' | '\n' | '"'] [_])* "\"") {
81            let newlines =
82                u32::try_from(s.chars().filter(|c| c == &'\n').count()).unwrap_or(u32::MAX);
83            (String::from(s).replace("\r\n", "\n").replace("\\\n", ""), newlines)
84        }
85
86        /// Parses a macro invocation: an optional `#`, a name, optional arguments.
87        pub rule macro_proper() -> Macro = quoted:"#"? n:name() args:arguments()? {
88            Macro {
89                name: n,
90                arguments: args,
91                original: String::new(),
92                quoted: quoted.is_some()
93            }
94        }
95
96        rule macro_token() -> Macro = original:$(macro_proper()) {?
97            parse_macro(original).map_err(|_| "a parseable macro invocation")
98        }
99
100        rule concat_token() = "##"
101
102        /// A `//` comment ends at a line end or at the end of the input (without one `CfgConvert` accepts it too).
103        rule sl_comment() -> u32 = "//" (!newline() [_])* &(newline() / ![_]) {
104            0
105        }
106
107        rule ml_comment() -> u32 = [' ' | '\t']* "/*" content:$((!"*/" (newline() / [_]))*) "*/" {
108            u32::try_from(content.chars().filter(|c| c == &'\n').count()).unwrap_or(u32::MAX)
109        }
110
111        /// An unterminated `/*` runs to the end of the file, as in `CfgConvert`. Only the parse of a whole
112        /// file (`file_end`) allows it, not the re-tokenised substrings of macro arguments.
113        rule open_comment() -> u32 = [' ' | '\t']* "/*" content:$([_]*) ![_] {?
114            if file_end {
115                // The file's final line end does not start another line: a diagnostic stays inside the file.
116                let content = content.strip_suffix('\n').unwrap_or(content);
117                Ok(u32::try_from(content.chars().filter(|c| c == &'\n').count()).unwrap_or(u32::MAX))
118            } else {
119                Err("a terminated comment")
120            }
121        }
122
123        rule comment_token() -> u32 = sl_comment() / ml_comment()
124
125        rule token() -> Token =
126            c:comment_token() { Token::CommentToken(c) } /
127            c:open_comment() { Token::OpenCommentToken(c) } /
128            sn:string_token() { Token::NewlineToken(sn.0, sn.1) } /
129            concat_token() { Token::ConcatToken } /
130            m:macro_token() { Token::MacroToken(m) } /
131            nm:nonmacro_token() { Token::RegularToken(nm) }
132
133        /// Parses a sequence of tokens.
134        pub rule tokens() -> Vec<Token> = t:(token()*) {
135            t
136        }
137
138        rule line() -> Line =
139            [' ' | '\t']* d:directive() cmts:(([' ' | '\t']* c:comment_token() { c })*) open:([' ' | '\t']* c:open_comment() { c })? [' ' | '\t']* {
140                Line::DirectiveLine(d, cmts.iter().sum::<u32>().saturating_add(open.unwrap_or(0)), open.is_some())
141            } /
142            [' ' | '\t']* t:tokens() { Line::TokenLine(t) }
143
144        /// Parses a whole file into lines.
145        pub rule file() -> Vec<Line> = lines:(line() ** (newline())) ![_] {
146            lines
147        }
148    }
149}
150
151/// Macro definition
152#[derive(Clone, Debug)]
153pub(crate) struct Definition {
154    name: String,
155    parameters: Option<Vec<String>>,
156    value: Vec<Token>,
157    local: bool,
158}
159
160/// Preprocessor directive
161#[derive(Debug)]
162#[expect(
163    clippy::enum_variant_names,
164    reason = "the variants are named after the directives they stand for"
165)]
166pub(crate) enum Directive {
167    /// `#include` directive containing the given path
168    IncludeDirective(String),
169    /// `#define` directive containing the definition
170    DefineDirective(Definition),
171    /// `#undef` directive containing the name of the macro
172    UndefDirective(String),
173    /// `#ifdef` directive containing the name of the macro
174    IfDefDirective(String),
175    /// `#ifndef` directive containing the name of the macro
176    IfNDefDirective(String),
177    /// `#else` directive
178    ElseDirective,
179    /// `#endif` directive
180    EndIfDirective,
181}
182
183/// Potential macro invocation
184#[derive(Debug)]
185pub(crate) struct Macro {
186    name: String,
187    arguments: Option<Vec<String>>,
188    original: String,
189    quoted: bool,
190}
191
192/// Preprocessor token
193#[derive(Debug)]
194#[expect(
195    clippy::enum_variant_names,
196    reason = "the variants are named after the tokens they stand for"
197)]
198pub(crate) enum Token {
199    /// Non-macro token
200    RegularToken(String),
201    /// Non-macro token that contains a number of newlines
202    NewlineToken(String, u32),
203    /// Potential macro token
204    MacroToken(Macro),
205    /// Comment token containing a number of newlines
206    CommentToken(u32),
207    /// Unterminated `/*` comment (to the end of the file) containing a number of newlines
208    OpenCommentToken(u32),
209    /// Token for the concatenation operator (`##`)
210    ConcatToken,
211}
212
213/// Preprocessor line
214#[derive(Debug)]
215pub(crate) enum Line {
216    /// Directive line
217    DirectiveLine(Directive, u32, bool),
218    /// Non-directive line of tokens
219    TokenLine(Vec<Token>),
220}
221
222/// The folders searched for absolute includes (`#include "\DZ\data\x.hpp"`).
223///
224/// A folder in `search` is searched recursively for a file whose path ends in the include path (and whose
225/// `$PBOPREFIX$` matches); the walk does not follow directory symlinks that leave the folder, and enters no directory twice;
226/// a search folder that reaches 1,000,000 visited entries is an error naming the
227/// include, and it ends the whole lookup (the other folders are not tried). A folder in `direct` is only
228/// checked for `<folder>/<include path>` (case-insensitively as a second pass), without walking it: use it for the
229/// current working directory, which can be a whole disk. The `search` folders are tried first, then the `direct`
230/// ones.
231#[derive(Debug, Clone, Default)]
232#[non_exhaustive]
233pub struct IncludeFolders {
234    /// Folders searched recursively.
235    pub search: Vec<PathBuf>,
236    /// Folders checked only for the include path directly below them.
237    pub direct: Vec<PathBuf>,
238    /// When set, includes come from these in-memory files only and the file system is never read (the
239    /// folders above are ignored); see [`MemoryIncludes`].
240    pub memory: Option<MemoryIncludes>,
241}
242
243/// The files of one archive (a PBO) as an include source, for input that must never reach the file system.
244///
245/// `origin` of the preprocessed input is the name of the including entry, and an include is resolved
246/// relative to the folder of that entry, or, in the engine's `\prefix\path` form, when it points into
247/// `prefix`. Absolute paths and drive letters are errors, as are empty components and control characters
248/// (see [`crate::pbo::safe_relative_path`]). In the relative form a `..` component is resolved inside the PBO and one that would leave
249/// the PBO's root is an error; in the `\prefix\path` form `..` is always an error. Names compare ignoring case. The files
250/// an `#include` pulls in count against a total of 64 MiB per preprocess call.
251#[derive(Debug, Clone, Default)]
252#[non_exhaustive]
253pub struct MemoryIncludes {
254    prefix: String,
255    /// lowercase `/`-separated name -> (name as given, content)
256    files: std::collections::BTreeMap<String, (String, Vec<u8>)>,
257    /// lowercase `/`-separated name -> why the entry is not available
258    unreadable: std::collections::BTreeMap<String, String>,
259}
260
261impl MemoryIncludes {
262    /// An include source over `files` (name, content) of an archive with the given `prefix` (may be empty).
263    /// The first of several entries with the same name (ignoring case and separator) wins.
264    #[must_use]
265    pub fn new(prefix: &str, files: Vec<(String, Vec<u8>)>) -> Self {
266        let mut map = std::collections::BTreeMap::new();
267        for (name, content) in files {
268            map.entry(name.replace('\\', "/").to_lowercase())
269                .or_insert((name, content));
270        }
271        MemoryIncludes {
272            prefix: prefix.replace('\\', "/").trim_matches('/').to_lowercase(),
273            files: map,
274            unreadable: std::collections::BTreeMap::new(),
275        }
276    }
277
278    /// Adds entries (name, reason) that exist in the archive but could not be read. An `#include` of one of
279    /// them fails with the reason instead of "not found". A name that is also a readable file stays readable.
280    #[must_use]
281    pub fn with_unreadable(mut self, entries: Vec<(String, String)>) -> Self {
282        for (name, reason) in entries {
283            self.unreadable
284                .entry(name.replace('\\', "/").to_lowercase())
285                .or_insert(reason);
286        }
287        self
288    }
289
290    /// The content of the entry `name` (ignoring case and separator), if it was read.
291    #[must_use]
292    pub fn get(&self, name: &str) -> Option<&[u8]> {
293        self.files
294            .get(&name.replace('\\', "/").to_lowercase())
295            .map(|(_, content)| content.as_slice())
296    }
297
298    /// The file `include_path` names when included from the entry `origin` (an entry name, or `None` for
299    /// the top level).
300    fn find(
301        &self,
302        include_path: &str,
303        origin: Option<&Path>,
304        line: u32,
305    ) -> Result<(&str, &[u8]), Error> {
306        let bad = |why: &str| {
307            error!(
308                "Include \"{}\" is not allowed ({why}); only files of the same PBO can be included.",
309                printable(include_path)
310            )
311        };
312        let key = if let Some(rest) = include_path.strip_prefix('\\') {
313            let rest = rest.replace('\\', "/").to_lowercase();
314            let below = if self.prefix.is_empty() {
315                None
316            } else {
317                rest.strip_prefix(&self.prefix)
318                    .and_then(|r| r.strip_prefix('/'))
319            };
320            let Some(below) = below else {
321                return Err(include_not_found(include_path, origin, line));
322            };
323            if crate::pbo::safe_relative_path(below).is_none() {
324                return Err(bad("invalid path"));
325            }
326            below.to_string()
327        } else {
328            let folder = origin
329                .map(|o| o.to_string_lossy().replace('\\', "/").to_lowercase())
330                .and_then(|o| o.rsplit_once('/').map(|(f, _)| f.to_string()));
331            let mut parts: Vec<String> = folder
332                .iter()
333                .flat_map(|f| f.split('/'))
334                .map(str::to_string)
335                .collect();
336            if include_path.starts_with('/') {
337                return Err(bad("absolute, drive or invalid path"));
338            }
339            for component in include_path.replace('\\', "/").to_lowercase().split('/') {
340                if component == ".." {
341                    if parts.pop().is_none() {
342                        return Err(bad("`..` leaves the PBO"));
343                    }
344                } else if crate::pbo::safe_relative_path(component).is_some() {
345                    parts.push(component.to_string());
346                } else {
347                    return Err(bad("absolute, drive or invalid path"));
348                }
349            }
350            parts.join("/")
351        };
352        if let Some((name, content)) = self.files.get(&key) {
353            return Ok((name.as_str(), content.as_slice()));
354        }
355        if let Some(reason) = self.unreadable.get(&key) {
356            return Err(match origin {
357                Some(o) => error!(
358                    "Include \"{}\" (from \"{}:{line}\") cannot be read: {reason}",
359                    printable(include_path),
360                    printable(&o.display().to_string())
361                ),
362                None => error!(
363                    "Include \"{}\" (line {line}) cannot be read: {reason}",
364                    printable(include_path)
365                ),
366            });
367        }
368        Err(include_not_found(include_path, origin, line))
369    }
370}
371
372impl IncludeFolders {
373    /// Folders searched recursively, none checked directly.
374    #[must_use]
375    pub fn new(search: Vec<PathBuf>) -> Self {
376        IncludeFolders {
377            search,
378            direct: Vec::new(),
379            memory: None,
380        }
381    }
382
383    /// Takes the includes from `memory` only; the file system is not read.
384    #[must_use]
385    pub fn with_memory(mut self, memory: MemoryIncludes) -> Self {
386        self.memory = Some(memory);
387        self
388    }
389
390    /// Adds folders that are only checked directly.
391    #[must_use]
392    pub fn with_direct(mut self, direct: Vec<PathBuf>) -> Self {
393        self.direct = direct;
394        self
395    }
396}
397
398/// Struct for additional information about preprocessor output. Contains import stack used for
399/// loop detection and the origins of all the lines in the output.
400#[derive(Debug)]
401pub struct PreprocessInfo {
402    /// For every line in the output, `line_origins` contains a line number (starting at 1) and a
403    /// `PathBuf` to the file where the line was found. The path may be `None` if the line was in the
404    /// original input to `preprocess` and `origin` was not given.
405    pub(crate) line_origins: Vec<(u32, Option<PathBuf>)>,
406    import_stack: Vec<PathBuf>,
407    /// Number of `#include`s currently being processed (the root file does not count).
408    include_depth: usize,
409    /// Tokens produced by macro expansion so far, over the file and its includes.
410    expanded: usize,
411    /// Bytes of token text copied by macro expansion so far.
412    expanded_bytes: usize,
413    /// Macro resolution steps taken so far, over the file and its includes.
414    steps: usize,
415    /// `#include`s processed so far, over the file and its includes.
416    includes: usize,
417    /// Bytes of the in-memory includes read so far.
418    include_bytes: usize,
419    /// Whether the last source line of the file is an entry line (not empty, a directive, a comment or
420    /// inside an unterminated comment), so that the line end the preprocessor adds is not one of the source.
421    pub(crate) ends_in_entry: bool,
422}
423
424impl PreprocessInfo {
425    /// For every line in the output, a line number (starting at 1) and the file where the line was
426    /// found. The path is `None` if the line was in the original input to `preprocess` and no
427    /// path was given.
428    #[must_use]
429    pub fn line_origins(&self) -> &[(u32, Option<PathBuf>)] {
430        &self.line_origins
431    }
432}
433
434/// Maximum nesting of brackets in the grammars: macro argument parentheses, config classes, config
435/// arrays and `__EVAL` parentheses and operator chains. Deeper input is a parse error, not a stack
436/// overflow. The generated parsers use up to about 5 KiB of stack per level in a debug build, so this
437/// fits the 1 MiB main-thread stack of Windows (`tests/nesting.rs` runs on such a stack); it equals the
438/// limit of the rapified reader (`MAX_RAPIFIED_DEPTH`), and real configs nest fewer than 20 levels.
439pub(crate) const MAX_NESTING_DEPTH: usize = 128;
440
441/// The error for input nested deeper than [`MAX_NESTING_DEPTH`].
442pub(crate) fn nesting_error() -> Error {
443    error!("Nesting is deeper than {MAX_NESTING_DEPTH} levels")
444}
445
446/// Runs a grammar entry point with a fresh nesting flag; a parse that hit the limit is the nesting
447/// error, whatever the parser reported.
448fn parse_checked<T, E>(
449    parse: impl FnOnce(&Cell<bool>) -> Result<T, E>,
450    on_error: impl FnOnce(E) -> Error,
451) -> Result<T, Error> {
452    let deep = Cell::new(false);
453    let result = parse(&deep);
454    if deep.get() {
455        return Err(nesting_error());
456    }
457    result.map_err(on_error)
458}
459
460/// Maximum nesting of macro expansions; deeper input is an error, not a stack overflow.
461const MAX_EXPANSION_DEPTH: usize = 64;
462
463/// Maximum tokens one source line may produce while its macros expand (every token copied at any level
464/// counts). Real configs stay far below it (see the #57 measurements in the README); an exponential
465/// definition chain hits it after a few milliseconds instead of exhausting memory.
466const MAX_LINE_EXPANSION: usize = 500_000;
467
468/// Maximum tokens all lines of one `preprocess` call (the includes count too) may produce.
469const MAX_FILE_EXPANSION: usize = 16_000_000;
470
471/// Maximum bytes of token text one source line may copy while its macros expand. Stringification
472/// collapses many tokens into one long string, so the token count alone does not bound memory.
473const MAX_LINE_BYTES: usize = 64 << 20;
474
475/// Maximum bytes copied by the lines of one `preprocess` call (the includes count too).
476const MAX_FILE_BYTES: usize = 512 << 20;
477
478// Both step caps are 100,000: real configs need at most 34 calls on a line and far fewer than 100,000 over a
479// file, so one number is easy to remember. The line cap still has a use: it stops a single line early (and names
480// it) instead of letting that line run up to the file cap.
481/// Maximum macro resolution steps (every call of a defined macro, argument pre-expansion included) of one source line. A step
482/// that produces nothing still costs, so expansion that drops its output cannot run exponentially (#59).
483const MAX_LINE_STEPS: usize = 100_000;
484
485/// Maximum macro resolution steps of all lines of one `preprocess` call (the includes count too). The line cap alone lets
486/// thousands of cheap-looking lines add up to tens of seconds (#64).
487const MAX_FILE_STEPS: usize = 100_000;
488
489/// Counts the tokens and bytes macro expansion produces, so that exponential expansion stops early.
490struct Budget {
491    tokens: usize,
492    bytes: usize,
493    steps: usize,
494    /// Steps the earlier lines of the `preprocess` call took.
495    file_steps: usize,
496}
497
498impl Budget {
499    /// A budget for one line; `file_steps` are the steps the earlier lines of the call already took.
500    fn new(file_steps: usize) -> Budget {
501        Budget {
502            tokens: 0,
503            bytes: 0,
504            steps: 0,
505            file_steps,
506        }
507    }
508
509    /// Charges one resolution step of the macro `name`.
510    fn step(&mut self, name: &str) -> Result<(), Error> {
511        self.steps = self.steps.saturating_add(1);
512        if self.steps > MAX_LINE_STEPS {
513            return Err(error!(
514                "Macro \"{}\" expands to more than {MAX_LINE_STEPS} macro calls in one line",
515                printable(name)
516            ));
517        }
518        if self.file_steps.saturating_add(self.steps) > MAX_FILE_STEPS {
519            return Err(error!(
520                "The file and its includes reach more than {MAX_FILE_STEPS} macro calls (at macro \"{}\")",
521                printable(name)
522            ));
523        }
524        self.spend(1, name.len().max(1), name)
525    }
526
527    /// Charges `tokens` tokens and `bytes` bytes of text, produced by the expansion of `name`.
528    fn spend(&mut self, tokens: usize, bytes: usize, name: &str) -> Result<(), Error> {
529        self.tokens = self.tokens.saturating_add(tokens);
530        self.bytes = self.bytes.saturating_add(bytes);
531        if self.tokens > MAX_LINE_EXPANSION || self.bytes > MAX_LINE_BYTES {
532            return Err(error!(
533                "Macro \"{}\" expands to more than {MAX_LINE_EXPANSION} tokens or {MAX_LINE_BYTES} bytes in one line",
534                printable(name)
535            ));
536        }
537        Ok(())
538    }
539}
540
541/// The bytes a token costs: its text, at least 1.
542fn token_bytes(token: &Token) -> usize {
543    match token {
544        Token::RegularToken(s) | Token::NewlineToken(s, _) => s.len().max(1),
545        Token::MacroToken(m) => m.original.len().max(1),
546        Token::CommentToken(_) | Token::OpenCommentToken(_) | Token::ConcatToken => 1,
547    }
548}
549
550/// Maximum nesting of `#include`s; deeper input is an error, not a stack overflow.
551const MAX_INCLUDE_DEPTH: usize = 32;
552
553/// Maximum `#include`s processed by one `preprocess` call, nested includes counted (every inclusion of
554/// the same file counts). Bounds exponential include fan-out (#60), which no token budget sees.
555const MAX_INCLUDES: usize = 10_000;
556
557/// Maximum bytes of in-memory includes ([`MemoryIncludes`]) one `preprocess` call reads, every inclusion counted.
558const MAX_INCLUDE_BYTES: usize = 64 << 20;
559
560/// The file as the cycle check compares it: canonical when the file system can say, else as given.
561fn identity(path: &Path, memory: bool) -> PathBuf {
562    if memory {
563        // an archive entry name is not a file system path
564        return PathBuf::from(path.to_string_lossy().replace('\\', "/").to_lowercase());
565    }
566    path.canonicalize().unwrap_or_else(|_| path.to_path_buf())
567}
568
569fn parse_macro(input: &str) -> Result<Macro, Error> {
570    let without_original = parse_checked(
571        |deep| preprocess_grammar::macro_proper(input, deep, false),
572        |e| error!("Failed to parse macro invocation \"{input}\": {e}"),
573    )?;
574
575    Ok(Macro {
576        original: String::from(input),
577        ..without_original
578    })
579}
580
581impl Clone for Macro {
582    fn clone(&self) -> Macro {
583        Macro {
584            name: self.name.clone(),
585            arguments: self.arguments.clone(),
586            original: self.original.clone(),
587            quoted: self.quoted,
588        }
589    }
590}
591
592impl Clone for Token {
593    fn clone(&self) -> Token {
594        match self {
595            Token::RegularToken(s) => Token::RegularToken(s.clone()),
596            Token::NewlineToken(s, n) => Token::NewlineToken(s.clone(), *n),
597            Token::MacroToken(m) => Token::MacroToken(m.clone()),
598            Token::CommentToken(n) => Token::CommentToken(*n),
599            Token::OpenCommentToken(n) => Token::OpenCommentToken(*n),
600            Token::ConcatToken => Token::ConcatToken,
601        }
602    }
603}
604
605impl Definition {
606    fn value(
607        &self,
608        arguments: Option<&[String]>,
609        def_map: &HashMap<String, Definition>,
610        stack: &[Definition],
611        depth: usize,
612        mismatches: &mut Vec<String>,
613        budget: &mut Budget,
614    ) -> Result<Option<Vec<Token>>, Error> {
615        let params = self.parameters.clone().unwrap_or_default();
616        let args = arguments.unwrap_or(&[]);
617
618        let empty_call = params.is_empty()
619            && args.len() == 1
620            && args.first().is_some_and(|a| a.trim().is_empty());
621        if params.len() != args.len() && !empty_call {
622            return Ok(None);
623        }
624
625        let mut tokens = self.value.clone();
626
627        if stack.iter().any(|d| d.name == self.name) {
628            return Ok(Some(tokens));
629        }
630
631        let mut stack_new: Vec<Definition> = stack.to_vec();
632        stack_new.push(self.clone());
633
634        if params.is_empty() {
635            tokens = Macro::resolve_all(&tokens, def_map, &stack_new, depth, mismatches, budget)?;
636        } else {
637            let mut local_map: HashMap<String, Definition> = HashMap::new();
638
639            for (key, value) in def_map {
640                local_map.insert(key.clone(), value.clone());
641            }
642
643            // @todo: handle these errors properly
644            for (param, arg) in params.iter().zip(args.iter()) {
645                let mut tokens = parse_checked(
646                    |deep| preprocess_grammar::tokens(arg, deep, false),
647                    |e| error!("Failed to parse macro argument: {e}"),
648                )?;
649                let stack: Vec<Definition> = Vec::new();
650                tokens = Macro::resolve_all(&tokens, def_map, &stack, depth, mismatches, budget)?;
651
652                local_map.insert(
653                    param.clone(),
654                    Definition {
655                        name: param.clone(),
656                        parameters: None,
657                        value: tokens,
658                        local: true,
659                    },
660                );
661            }
662
663            tokens =
664                Macro::resolve_all(&tokens, &local_map, &stack_new, depth, mismatches, budget)?;
665        }
666
667        Ok(Some(tokens))
668    }
669}
670
671impl Macro {
672    fn resolve_pseudoargs(
673        &self,
674        def_map: &HashMap<String, Definition>,
675        stack: &[Definition],
676        depth: usize,
677        mismatches: &mut Vec<String>,
678        budget: &mut Budget,
679    ) -> Result<Vec<Token>, Error> {
680        let mut tokens: Vec<Token> = Vec::new();
681        tokens.push(Token::RegularToken(self.name.clone()));
682
683        if self.arguments.is_none() {
684            return Ok(tokens);
685        }
686
687        let (_, without_name) = self.original.split_at(self.name.len());
688        let mut arg_tokens = parse_checked(
689            |deep| preprocess_grammar::tokens(without_name, deep, false),
690            |e| error!("Failed to parse macro arguments: {e}"),
691        )?;
692
693        arg_tokens = Macro::resolve_all(&arg_tokens, def_map, stack, depth, mismatches, budget)?;
694        for t in arg_tokens {
695            tokens.push(t);
696        }
697
698        Ok(tokens)
699    }
700
701    fn resolve(
702        &self,
703        def_map: &HashMap<String, Definition>,
704        stack: &[Definition],
705        depth: usize,
706        mismatches: &mut Vec<String>,
707        budget: &mut Budget,
708    ) -> Result<Vec<Token>, Error> {
709        // Every recursive path (body, argument pre-expansion, pseudo-arguments, local parameters) goes through
710        // here, so this one counter bounds them all; the `stack` check alone is reset by argument pre-expansion.
711        if depth >= MAX_EXPANSION_DEPTH {
712            return Err(error!(
713                "Macro \"{}\" expands too deep (limit {MAX_EXPANSION_DEPTH})",
714                printable(&self.name)
715            ));
716        }
717        let depth = depth + 1;
718        match def_map.get(&self.name) {
719            Some(def) => {
720                budget.step(&self.name)?;
721                if let Some(args) = &self.arguments {
722                    if def.parameters.is_none() {
723                        // Object-like macro followed by parentheses: expand it, keep the parentheses.
724                        let mut tokens = def
725                            .value(None, def_map, stack, depth, mismatches, budget)?
726                            .unwrap_or_default();
727                        let (_, rest) = self.original.split_at(self.name.len());
728                        let mut rest_tokens = parse_checked(
729                            |deep| preprocess_grammar::tokens(rest, deep, false),
730                            |e| error!("Failed to parse macro arguments: {e}"),
731                        )?;
732                        rest_tokens = Macro::resolve_all(
733                            &rest_tokens,
734                            def_map,
735                            stack,
736                            depth,
737                            mismatches,
738                            budget,
739                        )?;
740                        tokens.extend(rest_tokens);
741                        return Ok(tokens);
742                    }
743
744                    let expected = def.parameters.as_ref().map_or(0, std::vec::Vec::len);
745                    let got = if expected == 0
746                        && args.len() == 1
747                        && args.first().is_some_and(|a| a.trim().is_empty())
748                    {
749                        0
750                    } else {
751                        args.len()
752                    };
753                    if got != expected {
754                        let msg = format!(
755                            "Macro \"{}\" expects {} argument(s), got {}",
756                            printable(&self.name),
757                            expected,
758                            got
759                        );
760                        mismatches.push(msg);
761                        return Ok(if self.quoted {
762                            vec![Token::NewlineToken("\"\"".to_string(), 0)]
763                        } else {
764                            Vec::new()
765                        });
766                    }
767                }
768
769                let value = def.value(
770                    self.arguments.as_deref(),
771                    def_map,
772                    stack,
773                    depth,
774                    mismatches,
775                    budget,
776                )?;
777
778                if !def.local && self.quoted {
779                    // @todo: complain
780                }
781
782                if let Some(tokens) = value {
783                    if self.quoted {
784                        // charge before `concat` allocates the collapsed string
785                        budget.spend(
786                            tokens.len(),
787                            tokens.iter().map(token_bytes).sum(),
788                            &self.name,
789                        )?;
790                        let (concatted, newlines) = Token::concat(&tokens);
791                        let mut tokens: Vec<Token> = Vec::new();
792                        tokens.push(Token::NewlineToken(
793                            format!("\"{}\"", concatted.trim()),
794                            newlines,
795                        ));
796                        Ok(tokens)
797                    } else {
798                        Ok(tokens)
799                    }
800                } else {
801                    self.resolve_pseudoargs(def_map, stack, depth, mismatches, budget)
802                }
803            }
804            None => self.resolve_pseudoargs(def_map, stack, depth, mismatches, budget),
805        }
806    }
807
808    fn resolve_all(
809        tokens: &[Token],
810        def_map: &HashMap<String, Definition>,
811        stack: &[Definition],
812        depth: usize,
813        mismatches: &mut Vec<String>,
814        budget: &mut Budget,
815    ) -> Result<Vec<Token>, Error> {
816        let mut result: Vec<Token> = Vec::new();
817
818        for token in tokens {
819            if let Token::MacroToken(m) = token {
820                let resolved = m.resolve(def_map, stack, depth, mismatches, budget)?;
821                budget.spend(
822                    resolved.len(),
823                    resolved.iter().map(token_bytes).sum(),
824                    &m.name,
825                )?;
826                result.extend(resolved);
827            } else {
828                budget.spend(
829                    1,
830                    token_bytes(token),
831                    stack.last().map_or("", |d| d.name.as_str()),
832                )?;
833                result.push(token.clone());
834            }
835        }
836
837        Ok(result)
838    }
839}
840
841impl Token {
842    fn concat(tokens: &[Token]) -> (String, u32) {
843        let mut output = String::new();
844        let mut newlines = 0;
845
846        for token in tokens {
847            match token {
848                Token::RegularToken(s) => {
849                    output += s;
850                }
851                Token::NewlineToken(s, n) => {
852                    output += s;
853                    newlines += n;
854                }
855                Token::MacroToken(m) => {
856                    output += &m.original;
857                }
858                Token::CommentToken(n) | Token::OpenCommentToken(n) => {
859                    newlines += n;
860                }
861                Token::ConcatToken => {}
862            }
863        }
864
865        (output, newlines)
866    }
867}
868
869/// The source newlines of tokens, counted from the source and never from an expansion: `(inline,
870/// continuations)`. `inline` are those of strings and comments (a string inside a macro call is counted
871/// once), `continuations` the `\`-newlines outside strings, wherever they sit.
872fn source_newlines(tokens: &[Token]) -> (u32, u32) {
873    let (mut inline, mut continuations) = (0u32, 0u32);
874    for token in tokens {
875        match token {
876            Token::NewlineToken(_, n) | Token::CommentToken(n) | Token::OpenCommentToken(n) => {
877                inline = inline.saturating_add(*n);
878            }
879            Token::MacroToken(m) => {
880                let mut in_string = false;
881                let mut prev = ' ';
882                for c in m.original.chars() {
883                    match c {
884                        '"' => in_string = !in_string,
885                        '\n' if in_string => inline = inline.saturating_add(1),
886                        '\n' if prev == '\\' => continuations = continuations.saturating_add(1),
887                        _ => {}
888                    }
889                    prev = c;
890                }
891            }
892            Token::RegularToken(s) => {
893                let n = u32::try_from(s.matches("\\\n").count()).unwrap_or(u32::MAX);
894                continuations = continuations.saturating_add(n);
895            }
896            Token::ConcatToken => {}
897        }
898    }
899    (inline, continuations)
900}
901
902/// All source newlines of tokens that are not expanded.
903fn skipped_newlines(tokens: &[Token]) -> u32 {
904    let (inline, continuations) = source_newlines(tokens);
905    inline.saturating_add(continuations)
906}
907
908/// Reads the first line of a `$PBOPREFIX$` file.
909///
910/// # Errors
911///
912/// Returns an error if the file cannot be opened or read.
913fn read_prefix(prefix_path: &Path) -> Result<String, Error> {
914    let mut content = String::new();
915    File::open(prefix_path)?.read_to_string(&mut content)?;
916
917    Ok(content.lines().next().unwrap_or_default().to_string())
918}
919
920/// Returns the path seperator used on the current operating system
921#[must_use]
922pub(crate) fn pathsep() -> &'static str {
923    if cfg!(windows) { "\\" } else { "/" }
924}
925
926fn names_equal(a: &OsStr, b: &OsStr, ci: bool) -> bool {
927    if ci {
928        a.to_string_lossy()
929            .eq_ignore_ascii_case(&b.to_string_lossy())
930    } else {
931        a == b
932    }
933}
934
935/// Whether two paths to the same file differ in the case of any component.
936/// A case-insensitive filesystem (Windows) can open a path whose case does not
937/// match the file on disk; comparing the components byte-wise detects that.
938fn differs_in_case(a: &Path, b: &Path) -> bool {
939    let (mut a, mut b) = (a.components(), b.components());
940    loop {
941        match (a.next(), b.next()) {
942            (None, None) => return false,
943            (Some(x), Some(y)) if x.as_os_str() == y.as_os_str() => {}
944            _ => return true,
945        }
946    }
947}
948
949/// Resolves `relative` below `base`, comparing each component ASCII-case-insensitively.
950/// An exact match is preferred over a case-insensitive one at every level.
951///
952/// The names always come from the directory entries, never from `exists()`
953/// probes: on a case-insensitive filesystem (Windows) those succeed for any
954/// casing and would hide the name the file has on disk.
955fn resolve_case_insensitive(base: &Path, relative: &Path) -> Option<PathBuf> {
956    let mut current = base.to_path_buf();
957    for component in relative.components() {
958        let Component::Normal(name) = component else {
959            continue;
960        };
961        let mut exact = None;
962        let mut ignoring_case = None;
963        for entry in read_dir(&current).ok()?.filter_map(std::result::Result::ok) {
964            if exact.is_none() && names_equal(&entry.file_name(), name, false) {
965                exact = Some(entry);
966            } else if ignoring_case.is_none() && names_equal(&entry.file_name(), name, true) {
967                ignoring_case = Some(entry);
968            }
969        }
970        current = exact.or(ignoring_case)?.path();
971    }
972    Some(current)
973}
974
975fn matches_include_path(path: &Path, include_path: &str, ci: bool) -> bool {
976    let include_pathbuf = PathBuf::from(&include_path.replace('\\', pathsep()));
977
978    match (path.file_name(), include_pathbuf.file_name()) {
979        (Some(a), Some(b)) if names_equal(a, b, ci) => {}
980        _ => return false,
981    }
982
983    for parent in path.ancestors() {
984        if parent.is_file() {
985            continue;
986        }
987
988        let prefixpath = parent.join("$PBOPREFIX$");
989        if !prefixpath.is_file() {
990            continue;
991        }
992
993        // an unreadable prefix file cannot match the include; skip it instead of failing
994        let Ok(mut prefix) = read_prefix(&prefixpath) else {
995            continue;
996        };
997
998        prefix = if !prefix.is_empty() && !prefix.starts_with('\\') {
999            format!("\\{prefix}")
1000        } else {
1001            prefix
1002        };
1003
1004        let prefix_pathbuf = PathBuf::from(prefix.replace('\\', pathsep()));
1005
1006        let Ok(relative) = path.strip_prefix(parent) else {
1007            continue;
1008        };
1009        let test_path = prefix_pathbuf.join(relative);
1010
1011        let (mut a, mut b) = (test_path.components(), include_pathbuf.components());
1012        loop {
1013            match (a.next(), b.next()) {
1014                (None, None) => return true,
1015                (Some(x), Some(y)) if names_equal(x.as_os_str(), y.as_os_str(), ci) => {}
1016                _ => break,
1017            }
1018        }
1019    }
1020
1021    false
1022}
1023
1024/// Maximum directory entries one recursive include search visits below one folder before it gives up with an error
1025/// (#66). Real trees are far below it (a 109,000 entry `P:` drive copy); it stops a search that wandered into a
1026/// whole disk.
1027const MAX_SEARCH_ENTRIES: usize = 1_000_000;
1028
1029/// State of one recursive include search below `root`.
1030struct Walk<'a> {
1031    include_path: &'a str,
1032    root: &'a Path,
1033    ci: bool,
1034    visited: usize,
1035    limit: usize,
1036    /// Canonical paths of the directories entered so far: none is entered twice, so symlink loops and several
1037    /// links to one directory cost nothing.
1038    seen: std::collections::BTreeSet<PathBuf>,
1039}
1040
1041impl Walk<'_> {
1042    /// Whether the directory symlink `path` must not be followed: its target is outside the search root or cannot
1043    /// be resolved. (Loops are caught by the set of directories already entered.)
1044    fn skips_link(&self, path: &Path) -> bool {
1045        path.canonicalize()
1046            .map_or(true, |target| !target.starts_with(self.root))
1047    }
1048
1049    fn search(&mut self, directory: &Path) -> Result<Option<PathBuf>, Error> {
1050        // an unresolvable directory is skipped like an unreadable one
1051        let Ok(canonical) = directory.canonicalize() else {
1052            return Ok(None);
1053        };
1054        if !self.seen.insert(canonical) {
1055            return Ok(None);
1056        }
1057        // unreadable directories are skipped
1058        if let Ok(entries) = read_dir(directory) {
1059            for entry in entries.filter_map(std::result::Result::ok) {
1060                self.visited += 1;
1061                if self.visited > self.limit {
1062                    return Err(error!(
1063                        "Include \"{}\" not found: the search below \"{}\" stopped after {} entries",
1064                        printable(self.include_path),
1065                        printable(&self.root.display().to_string()),
1066                        self.limit
1067                    ));
1068                }
1069                let path = entry.path();
1070                let is_link = entry.file_type().is_ok_and(|t| t.is_symlink());
1071                if path.is_dir() {
1072                    if path.file_name() == Some(OsStr::new(".git")) {
1073                        continue;
1074                    }
1075                    if is_link && self.skips_link(&path) {
1076                        continue;
1077                    }
1078
1079                    if let Some(path) = self.search(&path)? {
1080                        return Ok(Some(path));
1081                    }
1082                } else if matches_include_path(&path, self.include_path, self.ci) {
1083                    return Ok(Some(path));
1084                }
1085            }
1086        }
1087
1088        Ok(lookup_directly(self.include_path, directory, self.ci))
1089    }
1090}
1091
1092/// The include `include_path` at exactly `directory/<include path>`, without looking at any other folder.
1093fn lookup_directly(include_path: &str, directory: &Path, ci: bool) -> Option<PathBuf> {
1094    let relative = PathBuf::from(
1095        include_path
1096            .trim_start_matches('\\')
1097            .replace('\\', pathsep()),
1098    );
1099    let direct_pathbuf = if ci {
1100        resolve_case_insensitive(directory, &relative)?
1101    } else {
1102        // A case-insensitive filesystem (Windows) opens the joined path even
1103        // when the case does not match; leave such matches to the ci pass,
1104        // which reports them, instead of returning them silently.
1105        let joined = directory.join(&relative);
1106        if !joined.is_file() {
1107            return None;
1108        }
1109        match resolve_case_insensitive(directory, &relative) {
1110            Some(found) if differs_in_case(&joined, &found) => return None,
1111            // the directories cannot be read, so the case cannot be checked
1112            _ => return Some(joined),
1113        }
1114    };
1115
1116    if direct_pathbuf.is_file() {
1117        return Some(direct_pathbuf);
1118    }
1119
1120    None
1121}
1122
1123fn canonicalize(path: &Path) -> PathBuf {
1124    let mut result = PathBuf::new();
1125    for component in path.components() {
1126        match component {
1127            Component::ParentDir => {
1128                result.pop();
1129            }
1130            _ => {
1131                result.push(component);
1132            }
1133        }
1134    }
1135    result
1136}
1137
1138fn include_not_found(include_path: &str, origin: Option<&Path>, line: u32) -> Error {
1139    match origin {
1140        Some(origin_path) => error!(
1141            "File \"{}\" included from \"{}:{}\" not found.",
1142            printable(include_path),
1143            printable(&origin_path.display().to_string()),
1144            line
1145        ),
1146        None => error!(
1147            "Included file \"{}\" (line {}) not found.",
1148            printable(include_path),
1149            line
1150        ),
1151    }
1152}
1153
1154fn warn_include_case(include_path: &str, found: &Path, warnings: &mut dyn WarningSink) {
1155    warnings.warn(Warning::new(
1156        Some("include-case"),
1157        format!(
1158            "Include \"{}\" only matched \"{}\" ignoring case.",
1159            printable(include_path),
1160            printable(&found.display().to_string())
1161        ),
1162    ));
1163}
1164
1165fn find_include_file(
1166    include_path: &str,
1167    origin: Option<&Path>,
1168    folders: &IncludeFolders,
1169    line: u32,
1170    warnings: &mut dyn WarningSink,
1171) -> Result<PathBuf, Error> {
1172    if include_path.starts_with('\\') {
1173        let canonical = |search_path: &PathBuf| {
1174            search_path.canonicalize().map_err(|e| {
1175                error!(
1176                    "Include folder \"{}\" is not usable: {}",
1177                    search_path.display(),
1178                    e
1179                )
1180            })
1181        };
1182        let roots = folders
1183            .search
1184            .iter()
1185            .map(canonical)
1186            .collect::<Result<Vec<PathBuf>, Error>>()?;
1187        let direct = folders
1188            .direct
1189            .iter()
1190            .map(canonical)
1191            .collect::<Result<Vec<PathBuf>, Error>>()?;
1192
1193        for ci in [false, true] {
1194            let mut found = None;
1195            for root in &roots {
1196                let mut walk = Walk {
1197                    include_path,
1198                    root,
1199                    ci,
1200                    visited: 0,
1201                    limit: MAX_SEARCH_ENTRIES,
1202                    seen: std::collections::BTreeSet::new(),
1203                };
1204                found = walk.search(root)?;
1205                if found.is_some() {
1206                    break;
1207                }
1208            }
1209            if found.is_none() {
1210                found = direct
1211                    .iter()
1212                    .find_map(|root| lookup_directly(include_path, root, ci));
1213            }
1214            if let Some(file_path) = found {
1215                if ci {
1216                    warn_include_case(include_path, &file_path, warnings);
1217                }
1218                return Ok(file_path);
1219            }
1220        }
1221
1222        Err(include_not_found(include_path, origin, line))
1223    } else {
1224        let mut path = PathBuf::from(include_path.replace('\\', pathsep()));
1225
1226        if let Some(origin_path) = origin {
1227            let absolute = PathBuf::from(origin_path).canonicalize()?;
1228            let Some(origin_dir) = absolute.parent() else {
1229                return Err(error!(
1230                    "Include origin \"{}\" has no parent directory.",
1231                    printable(&origin_path.display().to_string())
1232                ));
1233            };
1234            path = origin_dir.join(path);
1235        } else {
1236            path = current_dir()?.join(path);
1237        }
1238
1239        let absolute = canonicalize(&path);
1240        let root: PathBuf = absolute
1241            .components()
1242            .take_while(|c| !matches!(c, Component::Normal(_)))
1243            .collect();
1244        let relative: PathBuf = absolute
1245            .components()
1246            .skip_while(|c| !matches!(c, Component::Normal(_)))
1247            .collect();
1248
1249        if absolute.is_file() {
1250            // A case-insensitive filesystem (Windows) opens the probe even when
1251            // the case does not match the file on disk; check the components
1252            // against the directory entries so the warning fires there too.
1253            if let Some(found) = resolve_case_insensitive(&root, &relative)
1254                && differs_in_case(&absolute, &found)
1255            {
1256                warn_include_case(include_path, &found, warnings);
1257            }
1258            return Ok(absolute);
1259        }
1260
1261        // case-insensitive fallback for case-sensitive filesystems
1262        match resolve_case_insensitive(&root, &relative) {
1263            Some(found) if found.is_file() => {
1264                warn_include_case(include_path, &found, warnings);
1265                Ok(found)
1266            }
1267            _ => Err(include_not_found(include_path, origin, line)),
1268        }
1269    }
1270}
1271
1272/// The warning for a `/*` that is never closed (`line` is where the comment starts).
1273fn warn_unterminated(origin: Option<&Path>, line: u32, warnings: &mut dyn WarningSink) {
1274    let mut warning = Warning::new(
1275        Some("unterminated-comment"),
1276        "A /* comment is never closed; the rest of the file is ignored.",
1277    )
1278    .at_line(line);
1279    if let Some(origin) = origin {
1280        warning = warning.in_file(origin.display().to_string());
1281    }
1282    warnings.warn(warning);
1283}
1284
1285#[expect(
1286    clippy::too_many_lines,
1287    reason = "one branch per directive keeps the preprocessor loop readable"
1288)]
1289fn preprocess_rec(
1290    input: &str,
1291    origin: Option<&Path>,
1292    definition_map: &mut HashMap<String, Definition>,
1293    info: &mut PreprocessInfo,
1294    includefolders: &IncludeFolders,
1295    warnings: &mut dyn WarningSink,
1296) -> Result<String, Error> {
1297    // CRLF and classic Mac (lone CR) newlines
1298    let input = input.replace("\r\n", "\n").replace('\r', "\n");
1299    let deep = Cell::new(false);
1300    let parsed = preprocess_grammar::file(&input, &deep, true);
1301    if deep.get() {
1302        return Err(nesting_error());
1303    }
1304    let lines = PreprocessParseErrorExt::format_error(parsed, origin, &input)?;
1305    let mut output = String::new();
1306    let mut original_lineno = 1;
1307    let mut level = 0;
1308    let mut level_true = 0;
1309    let mut last_is_entry = false;
1310
1311    for line in lines {
1312        last_is_entry = false;
1313        // Every path leaves this block (never `continue`), so the line counter below always runs (#63).
1314        'line: {
1315            match line {
1316                Line::DirectiveLine(dir, newlines, open) => {
1317                    if open {
1318                        warn_unterminated(origin, original_lineno, warnings);
1319                    }
1320                    original_lineno += newlines;
1321
1322                    match dir {
1323                        Directive::IncludeDirective(path) => {
1324                            if level > level_true {
1325                                break 'line;
1326                            }
1327
1328                            let memory = includefolders.memory.as_ref();
1329                            let (file_path, in_memory) = if let Some(m) = memory {
1330                                let (name, content) = m.find(&path, origin, original_lineno)?;
1331                                (PathBuf::from(name), Some(content))
1332                            } else {
1333                                (
1334                                    find_include_file(
1335                                        &path,
1336                                        origin,
1337                                        includefolders,
1338                                        original_lineno,
1339                                        warnings,
1340                                    )?,
1341                                    None,
1342                                )
1343                            };
1344
1345                            let is_mem = memory.is_some();
1346                            let wanted = identity(&file_path, is_mem);
1347                            if info
1348                                .import_stack
1349                                .iter()
1350                                .any(|p| identity(p, is_mem) == wanted)
1351                            {
1352                                let chain: Vec<String> = info
1353                                    .import_stack
1354                                    .iter()
1355                                    .chain(std::iter::once(&file_path))
1356                                    .map(|p| printable(&p.display().to_string()))
1357                                    .collect();
1358                                return Err(error!("Include cycle: {}", chain.join(" -> ")));
1359                            }
1360                            if info.include_depth >= MAX_INCLUDE_DEPTH {
1361                                return Err(error!(
1362                                    "Include \"{}\" nests too deep (limit {MAX_INCLUDE_DEPTH})",
1363                                    printable(&path)
1364                                ));
1365                            }
1366
1367                            info.includes += 1;
1368                            if info.includes > MAX_INCLUDES {
1369                                return Err(error!(
1370                                    "Include \"{}\" is the {}th include; the limit is {MAX_INCLUDES} per preprocess call",
1371                                    printable(&path),
1372                                    info.includes
1373                                ));
1374                            }
1375
1376                            let content = if let Some(bytes) = in_memory {
1377                                info.include_bytes = info.include_bytes.saturating_add(bytes.len());
1378                                if info.include_bytes > MAX_INCLUDE_BYTES {
1379                                    return Err(error!(
1380                                        "Include \"{}\" brings the included files past {MAX_INCLUDE_BYTES} bytes",
1381                                        printable(&path)
1382                                    ));
1383                                }
1384                                String::from_utf8(bytes.to_vec()).map_err(|e| {
1385                                    error!(
1386                                        "Include \"{}\" is not valid UTF-8: {e}",
1387                                        printable(&path)
1388                                    )
1389                                })?
1390                            } else {
1391                                let mut content = String::new();
1392                                File::open(&file_path)?.read_to_string(&mut content)?;
1393                                content
1394                            };
1395                            info.import_stack.push(file_path.clone());
1396                            info.include_depth += 1;
1397                            let result = preprocess_rec(
1398                                &content,
1399                                Some(&file_path),
1400                                definition_map,
1401                                info,
1402                                includefolders,
1403                                warnings,
1404                            )
1405                            .prepend_error(format!(
1406                                "Failed to preprocess include \"{}\":",
1407                                printable(&path)
1408                            ))?;
1409
1410                            info.import_stack.pop();
1411                            info.include_depth -= 1;
1412
1413                            output += &result;
1414                        }
1415                        Directive::DefineDirective(def) => {
1416                            if let Some(at) = def
1417                                .value
1418                                .iter()
1419                                .position(|t| matches!(t, Token::OpenCommentToken(_)))
1420                            {
1421                                let before = def.value.get(..at).unwrap_or_default();
1422                                warn_unterminated(
1423                                    origin,
1424                                    original_lineno.saturating_add(skipped_newlines(before)),
1425                                    warnings,
1426                                );
1427                            }
1428                            original_lineno += skipped_newlines(&def.value);
1429
1430                            if level > level_true {
1431                                break 'line;
1432                            }
1433
1434                            if definition_map.remove(&def.name).is_some() {
1435                                // @todo: warn about redefine
1436                            }
1437
1438                            definition_map.insert(def.name.clone(), def);
1439                        }
1440                        Directive::UndefDirective(name) => {
1441                            if level > level_true {
1442                                break 'line;
1443                            }
1444
1445                            definition_map.remove(&name);
1446                        }
1447                        Directive::IfDefDirective(name) => {
1448                            level_true += i32::from(
1449                                level_true == level && definition_map.contains_key(&name),
1450                            );
1451                            level += 1;
1452                        }
1453                        Directive::IfNDefDirective(name) => {
1454                            level_true += i32::from(
1455                                level_true == level && !definition_map.contains_key(&name),
1456                            );
1457                            level += 1;
1458                        }
1459                        Directive::ElseDirective => {
1460                            if level_true + 1 == level {
1461                                level_true = level;
1462                            } else if level_true == level {
1463                                level_true -= 1;
1464                            }
1465                        }
1466                        Directive::EndIfDirective => {
1467                            if level == 0 {
1468                                return Err(error!("#endif without matching #ifdef/#ifndef"));
1469                            }
1470                            level -= 1;
1471                            if level_true > level {
1472                                level_true -= 1;
1473                            }
1474                        }
1475                    }
1476                }
1477                Line::TokenLine(tokens) => {
1478                    // Found after the parse, so a backtracked path never warns; also in a disabled branch,
1479                    // because the comment swallows the rest of the file there too.
1480                    if let Some(at) = tokens
1481                        .iter()
1482                        .position(|t| matches!(t, Token::OpenCommentToken(_)))
1483                    {
1484                        let before = tokens.get(..at).unwrap_or_default();
1485                        warn_unterminated(
1486                            origin,
1487                            original_lineno.saturating_add(skipped_newlines(before)),
1488                            warnings,
1489                        );
1490                    }
1491                    if level > level_true {
1492                        // A disabled line is never expanded (#61): it can neither error nor cost time.
1493                        original_lineno += skipped_newlines(&tokens);
1494                        break 'line;
1495                    }
1496                    last_is_entry = !tokens
1497                        .iter()
1498                        .any(|t| matches!(t, Token::OpenCommentToken(_)))
1499                        && tokens.iter().any(|t| match t {
1500                            Token::RegularToken(s) => !s.trim().is_empty(),
1501                            Token::NewlineToken(..) | Token::MacroToken(_) => true,
1502                            _ => false,
1503                        });
1504                    let stack: Vec<Definition> = Vec::new();
1505                    let mut mismatches: Vec<String> = Vec::new();
1506                    let mut budget = Budget::new(info.steps);
1507                    let resolved = Macro::resolve_all(
1508                        &tokens,
1509                        definition_map,
1510                        &stack,
1511                        0,
1512                        &mut mismatches,
1513                        &mut budget,
1514                    )
1515                    .prepend_error("Failed to resolve macros:");
1516                    let resolved = resolved?;
1517                    info.steps = info.steps.saturating_add(budget.steps);
1518                    info.expanded = info.expanded.saturating_add(budget.tokens);
1519                    info.expanded_bytes = info.expanded_bytes.saturating_add(budget.bytes);
1520                    if info.expanded > MAX_FILE_EXPANSION || info.expanded_bytes > MAX_FILE_BYTES {
1521                        return Err(error!(
1522                            "Macro expansion produced more than {MAX_FILE_EXPANSION} tokens or {MAX_FILE_BYTES} bytes in one file and its includes"
1523                        ));
1524                    }
1525
1526                    // Line numbers follow the source tokens, so a macro use never shifts them.
1527                    let (inline, continuations) = source_newlines(&tokens);
1528                    let (mut result, _) = Token::concat(&resolved);
1529                    result = result.replace("\r\n", "\n");
1530                    original_lineno += inline;
1531
1532                    result = result.replace("\\\n", "");
1533
1534                    let file = origin.map(|p| p.display().to_string());
1535                    for msg in mismatches {
1536                        let mut warning =
1537                            Warning::new(Some("macro-arg-count"), msg).at_line(original_lineno);
1538                        if let Some(file) = &file {
1539                            warning = warning.in_file(file.clone());
1540                        }
1541                        warnings.warn(warning);
1542                    }
1543
1544                    output += &result;
1545                    output += "\n";
1546
1547                    info.line_origins
1548                        .push((original_lineno, origin.map(Path::to_path_buf)));
1549                    original_lineno += continuations;
1550                }
1551            }
1552        }
1553        original_lineno += 1;
1554
1555        if level > 0 {
1556            // @todo: complain
1557        }
1558    }
1559
1560    info.ends_in_entry = last_is_entry;
1561    Ok(output)
1562}
1563
1564/// Reads input string and returns preprocessed string with an info struct containing the origins
1565/// of the lines in the output.
1566///
1567/// `path` is the path to the input if it is known and is used for relative includes and error
1568/// messages. `includefolders` are the folders searched recursively for absolute includes (see
1569/// [`IncludeFolders`] for folders that are only checked directly).
1570///
1571/// # Errors
1572///
1573/// Returns an error on parse failures, unreadable includes or invalid include folders.
1574///
1575/// # Examples
1576///
1577/// ```
1578/// # use dayzmake_core::preprocess::preprocess;
1579/// # use dayzmake_core::warning::Warnings;
1580/// let input = String::from("
1581/// #define QUOTE(x) #x
1582/// #define DOUBLES(x,y) x##_##y
1583///
1584/// foo = QUOTE(DOUBLES(abc, xyz));
1585/// ");
1586///
1587/// let (output, _) = preprocess(input, None, &Vec::new(), &mut Warnings::new()).expect("Failed to preprocess");
1588///
1589/// assert_eq!("foo = \"abc_xyz\";", output.trim());
1590/// ```
1591pub fn preprocess(
1592    input: String,
1593    origin: Option<PathBuf>,
1594    includefolders: &[PathBuf],
1595    warnings: &mut dyn WarningSink,
1596) -> Result<(String, PreprocessInfo), Error> {
1597    preprocess_in(
1598        input,
1599        origin,
1600        &IncludeFolders::new(includefolders.to_vec()),
1601        warnings,
1602    )
1603}
1604
1605/// Like [`preprocess`], with the include folders given as an [`IncludeFolders`], so that some of them can be
1606/// checked directly instead of searched recursively.
1607///
1608/// # Errors
1609///
1610/// Returns an error on parse failures, unreadable includes or invalid include folders.
1611#[expect(
1612    clippy::needless_pass_by_value,
1613    reason = "public API takes ownership of the input; callers pass owned strings"
1614)]
1615pub fn preprocess_in(
1616    mut input: String,
1617    origin: Option<PathBuf>,
1618    includefolders: &IncludeFolders,
1619    warnings: &mut dyn WarningSink,
1620) -> Result<(String, PreprocessInfo), Error> {
1621    if input.as_bytes().starts_with(&[0xef, 0xbb, 0xbf]) {
1622        input = input[3..].to_string();
1623    }
1624
1625    for folder in includefolders
1626        .search
1627        .iter()
1628        .chain(&includefolders.direct)
1629        .filter(|_| includefolders.memory.is_none())
1630    {
1631        if !folder.is_dir() {
1632            return Err(error!(
1633                "Include folder \"{}\" does not exist or is not a directory.",
1634                folder.display()
1635            ));
1636        }
1637    }
1638
1639    let mut info = PreprocessInfo {
1640        line_origins: Vec::new(),
1641        import_stack: Vec::new(),
1642        include_depth: 0,
1643        expanded: 0,
1644        expanded_bytes: 0,
1645        steps: 0,
1646        includes: 0,
1647        include_bytes: 0,
1648        ends_in_entry: false,
1649    };
1650
1651    if let Some(ref path) = origin {
1652        info.import_stack.push(path.clone());
1653    }
1654
1655    let mut def_map: HashMap<String, Definition> = HashMap::new();
1656
1657    match preprocess_rec(
1658        &input,
1659        origin.as_deref(),
1660        &mut def_map,
1661        &mut info,
1662        includefolders,
1663        warnings,
1664    ) {
1665        Ok(result) => Ok((result, info)),
1666        Err(e) => Err(e),
1667    }
1668}
1669
1670#[cfg(test)]
1671mod tests {
1672    use super::*;
1673
1674    /// Several links to one directory are entered once: the entries are counted once.
1675    #[cfg(unix)]
1676    #[test]
1677    fn sibling_links_are_not_visited_twice() {
1678        let dir = tempfile::tempdir().unwrap();
1679        let root = dir.path().canonicalize().unwrap();
1680        std::fs::create_dir(root.join("real")).unwrap();
1681        std::fs::write(root.join("real/f.txt"), "x").unwrap();
1682        std::os::unix::fs::symlink("real", root.join("l1")).unwrap();
1683        std::os::unix::fs::symlink("real", root.join("l2")).unwrap();
1684        let mut walk = Walk {
1685            include_path: "\\DZ\\gone.hpp",
1686            root: &root,
1687            ci: false,
1688            visited: 0,
1689            limit: 1000,
1690            seen: std::collections::BTreeSet::new(),
1691        };
1692        assert!(walk.search(&root).unwrap().is_none());
1693        // root: real, l1, l2 (3 entries) plus real's one file; the links add nothing
1694        assert_eq!(walk.visited, 4);
1695    }
1696
1697    /// The walk stops with an error naming the include once it visited more entries than its limit (#66).
1698    #[test]
1699    fn search_stops_after_the_entry_limit() {
1700        let dir = tempfile::tempdir().unwrap();
1701        for i in 0..10 {
1702            std::fs::write(dir.path().join(format!("f{i}.txt")), "x").unwrap();
1703        }
1704        let root = dir.path().canonicalize().unwrap();
1705        let mut walk = Walk {
1706            include_path: "\\DZ\\gone.hpp",
1707            root: &root,
1708            ci: false,
1709            visited: 0,
1710            limit: 5,
1711            seen: std::collections::BTreeSet::new(),
1712        };
1713        let err = walk.search(&root).unwrap_err().to_string();
1714        assert!(
1715            err.contains("DZ") && err.contains("stopped after 5 entries"),
1716            "{err}"
1717        );
1718    }
1719
1720    use std::ffi::OsStr;
1721
1722    /// Argument-count mismatches go into the buffer of the call, not into state shared by the
1723    /// thread: a second resolve with a fresh buffer sees nothing of the first.
1724    #[test]
1725    fn arg_count_mismatches_belong_to_their_call() {
1726        let mut defs: HashMap<String, Definition> = HashMap::new();
1727        let def = Definition {
1728            name: "F".to_string(),
1729            parameters: Some(vec!["a".to_string()]),
1730            value: Vec::new(),
1731            local: false,
1732        };
1733        defs.insert("F".to_string(), def);
1734        let tokens = vec![Token::MacroToken(parse_macro("F(1,2)").unwrap())];
1735
1736        let mut first = Vec::new();
1737        Macro::resolve_all(&tokens, &defs, &[], 0, &mut first, &mut Budget::new(0)).unwrap();
1738        assert_eq!(first.len(), 1);
1739
1740        let plain = vec![Token::RegularToken("x".to_string())];
1741        let mut second = Vec::new();
1742        Macro::resolve_all(&plain, &defs, &[], 0, &mut second, &mut Budget::new(0)).unwrap();
1743        assert_eq!(second.len(), 0);
1744    }
1745
1746    /// The resolver reports the name the file has on disk, never the requested
1747    /// one: on Windows an `exists()` probe succeeds for any casing, so only the
1748    /// directory entries carry the truth.
1749    #[test]
1750    fn resolve_reports_the_on_disk_casing() {
1751        let dir = tempfile::tempdir().unwrap();
1752        std::fs::write(dir.path().join("inc.hpp"), b"x").unwrap();
1753        let found = resolve_case_insensitive(dir.path(), Path::new("INC.hpp")).unwrap();
1754        assert_eq!(found.file_name(), Some(OsStr::new("inc.hpp")));
1755    }
1756
1757    /// An exact match is preferred over a case-insensitive one. The fixture
1758    /// needs two files that differ only in case, which a case-insensitive
1759    /// filesystem (Windows) cannot hold.
1760    #[test]
1761    #[cfg(not(windows))]
1762    fn resolve_prefers_the_exact_match() {
1763        let dir = tempfile::tempdir().unwrap();
1764        std::fs::write(dir.path().join("Inc.hpp"), b"a").unwrap();
1765        std::fs::write(dir.path().join("inc.hpp"), b"b").unwrap();
1766        let found = resolve_case_insensitive(dir.path(), Path::new("inc.hpp")).unwrap();
1767        assert_eq!(found.file_name(), Some(OsStr::new("inc.hpp")));
1768    }
1769
1770    #[test]
1771    fn case_difference_is_detected_component_wise() {
1772        assert!(differs_in_case(
1773            Path::new("/a/B.hpp"),
1774            Path::new("/a/b.hpp")
1775        ));
1776        assert!(!differs_in_case(
1777            Path::new("/a/b.hpp"),
1778            Path::new("/a/b.hpp")
1779        ));
1780    }
1781}