% References for "Who May Read the Web?" (technical report 02, version 1.0).
% Every entry is cited in the manuscript. Web sources carry the date on which they were read.
% Keys: author year keyword, lower case.

@techreport{rfc9309,
  author = {Koster, Martijn and Illyes, Gary and Zeller, Henner and Sassman, Lizzi},
  title = {Robots Exclusion Protocol},
  institution = {RFC Editor},
  year = {2022},
  howpublished = {RFC 9309, Proposed Standard},
  url = {https://www.rfc-editor.org/rfc/rfc9309.txt},
  note = {Published September 2022; read in full}
}

@misc{google2026robotsspec,
  author = {{Google}},
  title = {How Google interprets the robots.txt specification},
  howpublished = {Google Search Central documentation},
  year = {2026},
  url = {https://developers.google.com/crawling/docs/robots-txt/robots-txt-spec},
  note = {Page last updated 2026-08-31. Retrieved 2026-10-04}
}

@misc{googlerobotstxt,
  author = {{Google}},
  title = {Robots.txt parser and matcher library (google/robotstxt)},
  howpublished = {Source repository, Apache-2.0 licence},
  year = {2026},
  url = {https://github.com/google/robotstxt},
  note = {Commit 22b355ff used for the parser comparison. Retrieved 2026-10-04}
}

@inproceedings{liu2025somesite,
  author = {Liu, Enze and Luo, Elisa and Shan, Shawn and Voelker, Geoffrey M. and Zhao, Ben Y. and Savage, Stefan},
  title = {Somesite {I} Used To Crawl: Awareness, Agency and Efficacy in Protecting Content Creators From {AI} Crawlers},
  booktitle = {Proceedings of the 2025 ACM Internet Measurement Conference (IMC '25)},
  year = {2025},
  pages = {78--99},
  doi = {10.1145/3730567.3732913}
}

@inproceedings{kim2025scrapers,
  author = {Kim, Taein and Bock, Karstan and Luo, Claire and Liswood, Amanda and Poroslay, Chloe and Wenger, Emily},
  title = {Scrapers Selectively Respect robots.txt Directives: Evidence from a Large-Scale Empirical Study},
  booktitle = {Proceedings of the 2025 ACM Internet Measurement Conference (IMC '25)},
  year = {2025},
  pages = {541--557},
  doi = {10.1145/3730567.3764471}
}

@misc{bouchaud2025restrictions,
  author = {Bouchaud, Paul and Ramaciotti, Pedro},
  title = {Web Crawler Restrictions, {AI} Training Datasets \& Political Biases},
  howpublished = {arXiv:2510.09031, preprint},
  year = {2025},
  url = {https://arxiv.org/abs/2510.09031},
  note = {Version 1, 10 October 2025}
}

@misc{steinacker2026misinformation,
  author = {Steinacker-Olsztyn, Nicolas and Gosain, Devashish and Dao, Ha},
  title = {Is Misinformation More Open? {A} Study of robots.txt Gatekeeping on the Web},
  howpublished = {arXiv:2510.10315, version 4; the PDF carries an ACM block for The Web Conference 2026 whose DOI could not be resolved through Crossref on 2026-10-04},
  year = {2026},
  url = {https://arxiv.org/abs/2510.10315}
}

@article{hoffmann2026aitxt,
  author = {Hoffmann, Felix and Goergens, Tobias and Khosla, Sonal and Bajpai, Vaibhav},
  title = {From robots.txt to ai.txt: Mapping the Evolution of Web Permissions in the Age of {AI}},
  journal = {ACM SIGCOMM Computer Communication Review},
  year = {2026},
  volume = {56},
  number = {2},
  pages = {29--41},
  doi = {10.1145/3831956.3831960},
  note = {Only the abstract and the published code and data artefact (github.com/felixhoffmnn/robots-ai-permissions) were read for this report}
}

@inproceedings{longpre2024consent,
  author = {Longpre, Shayne and Mahari, Robert and Lee, Ariel and Lund, Campbell and others},
  title = {Consent in Crisis: The Rapid Decline of the {AI} Data Commons},
  booktitle = {Advances in Neural Information Processing Systems 37 (NeurIPS 2024), Datasets and Benchmarks Track},
  year = {2024},
  doi = {10.52202/079017-3431}
}

@techreport{fletcher2024newsblock,
  author = {Fletcher, Richard},
  title = {How Many News Websites Block {AI} Crawlers?},
  institution = {Reuters Institute for the Study of Journalism},
  year = {2024},
  howpublished = {Factsheet, February 2024},
  doi = {10.60625/risj-xm9g-ws87}
}

@misc{dinzinger2024survey,
  author = {Dinzinger, Michael and Heß, Florian and Granitzer, Michael},
  title = {A Survey of Web Content Control for Generative {AI}},
  howpublished = {arXiv:2404.02309, preprint},
  year = {2024},
  url = {https://arxiv.org/abs/2404.02309}
}

@inproceedings{sun2007robots,
  author = {Sun, Yang and Zhuang, Ziming and Giles, C. Lee},
  title = {A Large-Scale Study of Robots.txt},
  booktitle = {Proceedings of the 16th International Conference on World Wide Web (WWW 2007)},
  year = {2007},
  pages = {1123--1124},
  url = {https://archives.iw3c2.org/www2007/posters/poster1034.pdf}
}

@inproceedings{kolay2008robots,
  author = {Kolay, Santanu and D'Alberto, Paolo and Dasdan, Ali and Bhattacharjee, Arnab},
  title = {A Larger Scale Study of Robots.txt},
  booktitle = {Proceedings of the 17th International Conference on World Wide Web (WWW 2008)},
  year = {2008},
  pages = {1171--1172},
  url = {https://archives.iw3c2.org/www2008/papers/pdf/p1171-kolay.pdf}
}

@incollection{almanac2025seo,
  author = {Chulwuma, Amaka and Green, Chris and Brannon, Sophie},
  title = {{SEO}},
  booktitle = {The 2025 Web Almanac, chapter 5},
  publisher = {HTTP Archive},
  year = {2026},
  url = {https://almanac.httparchive.org/en/2025/seo},
  note = {Published 15 January 2026, updated 9 June 2026. Retrieved 2026-10-04}
}

@incollection{almanac2025genai,
  author = {Liebel, Christian and Vekaria, Yash and Pagel, Jonathan},
  title = {Generative {AI}},
  booktitle = {The 2025 Web Almanac, chapter 4},
  publisher = {HTTP Archive},
  year = {2026},
  url = {https://almanac.httparchive.org/en/2025/generative-ai},
  note = {Published 15 January 2026. Retrieved 2026-10-04}
}

@misc{cloudflare2025radarrobots,
  author = {{Cloudflare}},
  title = {Expanded {AI} insights in Cloudflare Radar: robots.txt directives for {AI} user agents},
  howpublished = {Cloudflare developer changelog, 4 February 2025, and the Radar API endpoint radar/robots\_txt/top/user\_agents/directive},
  year = {2025},
  url = {https://developers.cloudflare.com/changelog/post/2025-02-04-radar-ai-insights/},
  note = {Weekly series; the comparison table uses the scan dated 2026-10-05. Retrieved 2026-10-04}
}

@misc{cloudflare2026agentreadiness,
  author = {Jesus, André and Morrison, Vance},
  title = {Introducing the Agent Readiness score. Is your site agent-ready?},
  howpublished = {Cloudflare Blog, 17 April 2026, with the Radar agent-readiness API},
  year = {2026},
  url = {https://blog.cloudflare.com/agent-readiness/},
  note = {Cloudflare proposed the Content Signals vocabulary and writes robots.txt text for its customers. Retrieved 2026-10-04}
}

@techreport{aivisibility2026q4,
  author = {McNeece, Mark},
  title = {{AI} Discovery File Adoption Research, {Q4} 2026},
  institution = {AI Visibility},
  year = {2026},
  url = {https://www.ai-visibility.org.uk/research/2026-q4/},
  note = {Published 1 October 2026 with a companion post titled One in Ten Top Websites Now Has an AI Discovery File. The vendor defines the file set it measures. Retrieved 2026-10-04}
}

@misc{rankability2026llms,
  author = {{Rankability}},
  title = {llms.txt Adoption Research Report},
  howpublished = {Vendor research page, 18 September 2026},
  year = {2026},
  url = {https://www.rankability.com/blog/llms-txt-adoption/},
  note = {An SEO vendor's scan of Tranco list V3YPN. Retrieved 2026-10-04}
}

@misc{seomator2026llms,
  author = {{SEOmator}},
  title = {llms.txt: We Scanned the Web's Top 100,000 Domains},
  howpublished = {Vendor blog, 13 September 2026},
  year = {2026},
  url = {https://seomator.com/blog/llms-txt-adoption},
  note = {An SEO vendor's scan of the Cloudflare Radar top 100,000. Retrieved 2026-10-04}
}

@misc{sauerburger2026llms,
  author = {Sauerburger, Frank},
  title = {llms.txt adoption, one year later},
  howpublished = {Personal blog, 8 September 2026},
  year = {2026},
  url = {https://frank.sauerburger.io/2026/09/08/llms-txt-adoption-one-year-later.html},
  note = {Retrieved 2026-10-04}
}

@misc{apievangelist2026a2a,
  author = {Lane, Kin},
  title = {{A2A} Adoption Is 421 of 27,840, and Most of Them Arrived With the Card},
  howpublished = {API Evangelist, 21 September 2026},
  year = {2026},
  url = {https://apievangelist.com/2026/09/21/a2a-adoption-is-421-of-27840-and-most-of-them-arrived-with-the-card/},
  note = {The earlier post of 29 July 2026, Most Published Agent Cards Are Not Actually A2A, was read as background. Retrieved 2026-10-04}
}

@misc{originality2024tracker,
  author = {Gillham, Jonathan},
  title = {Block {AI} Bots from Crawling Websites Using Robots.txt},
  howpublished = {Originality.ai, 22 August 2024, with a live dashboard},
  year = {2024},
  url = {https://originality.ai/ai-bot-blocking},
  note = {The list and the blocking rule behind the dashboard are not documented. Retrieved 2026-10-04}
}

@misc{knownagents2026index,
  author = {{Known Agents}},
  title = {The Agentic Web Index},
  howpublished = {Known Agents (formerly Dark Visitors), live page},
  year = {2026},
  url = {https://knownagents.com/insights},
  note = {Blocking rule not documented. Retrieved 2026-10-04}
}

@misc{knownagents2026generator,
  author = {{Known Agents}},
  title = {Automatic Robots.txt},
  howpublished = {Known Agents documentation: generated per-agent robots.txt and REST API},
  year = {2026},
  url = {https://knownagents.com/docs/robots-txt},
  note = {Retrieved 2026-10-04}
}

@misc{welsh2023newshomepages,
  author = {Welsh, Ben},
  title = {Who blocks {OpenAI}, {Google} {AI} and {Common Crawl}?},
  howpublished = {News Homepages project documentation, live page},
  year = {2026},
  url = {https://palewi.re/docs/news-homepages/openai-gptbot-robotstxt.html},
  note = {Live page; its footer read Last updated October 4, 2026. Retrieved 2026-10-04}
}

@techreport{tollbit2026h1,
  author = {{TollBit}},
  title = {State of the Bots, 2026 Q1 \& Q2: The Bad Bots},
  institution = {TollBit},
  year = {2026},
  url = {https://tollbit.com/state-of-the-bots/},
  note = {A vendor of bot paywalls reporting on its own publisher customers. Retrieved 2026-10-04}
}

@inproceedings{lepochat2019tranco,
  author = {Le Pochat, Victor and Van Goethem, Tom and Tajalizadehkhoob, Samaneh and Korczyński, Maciej and Joosen, Wouter},
  title = {Tranco: A Research-Oriented Top Sites Ranking Hardened Against Manipulation},
  booktitle = {Proceedings of the 26th Annual Network and Distributed System Security Symposium (NDSS 2019)},
  year = {2019},
  doi = {10.14722/ndss.2019.23386}
}

@inproceedings{ruth2022toplists,
  author = {Ruth, Kimberly and Kumar, Deepak and Wang, Brandon and Valenta, Luke and Durumeric, Zakir},
  title = {Toppling Top Lists: Evaluating the Accuracy of Popular Website Lists},
  booktitle = {Proceedings of the 22nd ACM Internet Measurement Conference (IMC '22)},
  year = {2022},
  pages = {374--387},
  doi = {10.1145/3517745.3561444}
}

@inproceedings{scheitle2018toplists,
  author = {Scheitle, Quirin and others},
  title = {A Long Way to the Top: Significance, Structure, and Stability of Internet Top Lists},
  booktitle = {Proceedings of the Internet Measurement Conference 2018 (IMC '18)},
  year = {2018},
  pages = {478--493},
  doi = {10.1145/3278532.3278574}
}

@inproceedings{vallina2020classification,
  author = {Vallina, Pelayo and Le Pochat, Victor and Feal, Álvaro and others},
  title = {Mis-shapes, Mistakes, Misfits: An Analysis of Domain Classification Services},
  booktitle = {Proceedings of the ACM Internet Measurement Conference (IMC '20)},
  year = {2020},
  pages = {598--618},
  doi = {10.1145/3419394.3423660}
}

@inproceedings{ahmad2020crawlers,
  author = {Ahmad, Syed Suleman and Dar, Muhammad Daniyal and Zaffar, Muhammad Fareed and Vallina-Rodriguez, Narseo and Nithyanand, Rishab},
  title = {Apophanies or Epiphanies? How Crawlers Impact Our Understanding of the Web},
  booktitle = {Proceedings of The Web Conference 2020 (WWW '20)},
  year = {2020},
  pages = {271--280},
  doi = {10.1145/3366423.3380113}
}

@inproceedings{jueckstock2021realistic,
  author = {Jueckstock, Jordan and others},
  title = {Towards Realistic and Reproducible Web Crawl Measurements},
  booktitle = {Proceedings of the Web Conference 2021 (WWW '21)},
  year = {2021},
  pages = {80--91},
  doi = {10.1145/3442381.3450050}
}

@techreport{menlo2012,
  author = {Dittrich, David and Kenneally, Erin},
  title = {The Menlo Report: Ethical Principles Guiding Information and Communication Technology Research},
  institution = {U.S. Department of Homeland Security},
  year = {2012},
  url = {https://www.caida.org/catalog/papers/2012_menlo_report_actual_formatted/},
  note = {August 2012}
}

@article{partridge2016ethics,
  author = {Partridge, Craig and Allman, Mark},
  title = {Ethical considerations in network measurement papers},
  journal = {Communications of the ACM},
  year = {2016},
  volume = {59},
  number = {10},
  pages = {58--64},
  doi = {10.1145/2896816}
}

@inproceedings{durumeric2013zmap,
  author = {Durumeric, Zakir and Wustrow, Eric and Halderman, J. Alex},
  title = {{ZMap}: Fast Internet-Wide Scanning and Its Security Applications},
  booktitle = {Proceedings of the 22nd USENIX Security Symposium},
  year = {2013},
  url = {https://zmap.io/paper.pdf}
}

@inproceedings{hantke2024redlines,
  author = {Hantke, Florian and Roth, Sebastian and Mrowczynski, Rafael and Utz, Christine and Stock, Ben},
  title = {Where Are the Red Lines? Towards Ethical Server-Side Scans in Security and Privacy Research},
  booktitle = {Proceedings of the 2024 IEEE Symposium on Security and Privacy (S\&P 2024)},
  year = {2024},
  pages = {4405--4423},
  doi = {10.1109/SP54263.2024.00104}
}

@article{brown2001interval,
  author = {Brown, Lawrence D. and Cai, T. Tony and DasGupta, Anirban},
  title = {Interval Estimation for a Binomial Proportion},
  journal = {Statistical Science},
  year = {2001},
  volume = {16},
  number = {2},
  pages = {101--133},
  doi = {10.1214/ss/1009213286}
}

@techreport{rfc9727,
  author = {Smith, Kevin},
  title = {api-catalog: A Well-Known {URI} and Link Relation to Help Discovery of {APIs}},
  institution = {RFC Editor},
  year = {2025},
  howpublished = {RFC 9727, Proposed Standard},
  url = {https://www.rfc-editor.org/rfc/rfc9727.txt},
  note = {Published June 2025; the path is a permanent IANA registration}
}

@techreport{rfc9728,
  author = {Jones, Michael B. and Hunt, Phil and Parecki, Aaron},
  title = {{OAuth} 2.0 Protected Resource Metadata},
  institution = {RFC Editor},
  year = {2025},
  howpublished = {RFC 9728, Proposed Standard},
  url = {https://www.rfc-editor.org/rfc/rfc9728.txt},
  note = {Published April 2025}
}

@techreport{rfc8414,
  author = {Jones, Michael B. and Sakimura, Nat and Bradley, John},
  title = {{OAuth} 2.0 Authorization Server Metadata},
  institution = {RFC Editor},
  year = {2018},
  howpublished = {RFC 8414, Proposed Standard},
  url = {https://www.rfc-editor.org/rfc/rfc8414.txt},
  note = {Published June 2018}
}

@techreport{rfc9116,
  author = {Foudil, Edwin and Shafranovich, Yakov},
  title = {A File Format to Aid in Security Vulnerability Disclosure},
  institution = {RFC Editor},
  year = {2022},
  howpublished = {RFC 9116, Informational},
  url = {https://www.rfc-editor.org/rfc/rfc9116.txt},
  note = {Published April 2022}
}

@misc{iana2026wellknown,
  author = {{Internet Assigned Numbers Authority}},
  title = {Well-Known {URIs} registry},
  howpublished = {IANA protocol registry},
  year = {2026},
  url = {https://www.iana.org/assignments/well-known-uris/},
  note = {Registry last updated 2026-09-16, 105 entries. Retrieved 2026-10-04}
}

@misc{a2a2026spec,
  author = {{A2A Project}},
  title = {Agent2Agent ({A2A}) Protocol Specification, version 1.0},
  howpublished = {Specification, Linux Foundation project; releases v1.0.0 (2026-03-12) and v1.0.1 (2026-05-28)},
  year = {2026},
  url = {https://a2a-protocol.org/latest/specification/},
  note = {The Agent Card path was renamed from agent.json in v0.3.0 (2025-07-30). Retrieved 2026-10-04}
}

@misc{mcp2026spec,
  author = {{Model Context Protocol}},
  title = {Model Context Protocol: specification revision 2026-07-28, and the server-card discovery proposal {SEP-2127}},
  howpublished = {Specification repository and proposal, under review on 2026-10-04},
  year = {2026},
  url = {https://github.com/modelcontextprotocol/modelcontextprotocol},
  note = {The specification defines no well-known discovery file; the proposal routes discovery through /.well-known/ai-catalog.json. Retrieved 2026-10-04}
}

@misc{ard2026spec,
  author = {Bu, Junjie and Guha, R. V. and Smith, Shaun},
  title = {Agentic Resource Discovery Specification},
  howpublished = {Version 0.91, status Proposal, 26 August 2026},
  year = {2026},
  url = {https://agenticresourcediscovery.org/spec/},
  note = {Retrieved 2026-10-04}
}

@misc{agentskills2026,
  author = {{Cloudflare}},
  title = {Agent Skills Discovery via Well-Known {URIs}},
  howpublished = {Draft specification, version 0.2.0, source repository},
  year = {2026},
  url = {https://github.com/cloudflare/agent-skills-discovery-rfc},
  note = {Retrieved 2026-10-04}
}

@misc{llmstxt2026,
  author = {Howard, Jeremy},
  title = {The /llms.txt file, version 2},
  howpublished = {Proposal page, llmstxt.org},
  year = {2026},
  url = {https://llmstxt.org/},
  note = {Published 2024-09-03, revised 2026-08-10. The page does not define llms-full.txt. Retrieved 2026-10-04}
}

@misc{agentsjson2025,
  author = {{Wildcard}},
  title = {agents.json specification, version 0.1.0},
  howpublished = {Source repository README},
  year = {2025},
  url = {https://github.com/wild-card-ai/agents-json},
  note = {Repository last pushed 2025-08-21. Retrieved 2026-10-04}
}

@misc{openai2026plugins,
  author = {{OpenAI}},
  title = {Plugin development for {ChatGPT} and {Codex}},
  howpublished = {OpenAI developer documentation},
  year = {2026},
  url = {https://developers.openai.com/plugins},
  note = {The current pages do not describe ai-plugin.json; the earlier plugin documentation redirects here. Retrieved 2026-10-04}
}

@misc{spawning2023aitxt,
  author = {{Spawning}},
  title = {ai.txt generator},
  howpublished = {Spawning website},
  year = {2023},
  url = {https://site.spawning.ai/spawning-ai-txt},
  note = {The generator page carries a 2023 footer; the project's main site showed a maintenance page and answered /ai.txt with an HTML page on 2026-10-04}
}

@misc{contentsignals2026site,
  author = {{Cloudflare}},
  title = {Content Signals},
  howpublished = {contentsignals.org},
  year = {2026},
  url = {https://contentsignals.org/},
  note = {Undated page describing the Content-Signal record syntax. Retrieved 2026-10-04}
}

@misc{cloudflare2025contentsignals,
  author = {Allen, Will and others},
  title = {Giving users choice with Cloudflare's new Content Signals Policy},
  howpublished = {Cloudflare Blog, 24 September 2025},
  year = {2025},
  url = {https://blog.cloudflare.com/content-signals-policy/},
  note = {Modified 2026-07-15. Retrieved 2026-10-04}
}

@misc{aipref2026vocab,
  author = {Keller, Paul and Thomson, Martin},
  title = {A Vocabulary For Expressing {AI} Usage Preferences},
  howpublished = {Internet-Draft draft-ietf-aipref-vocab-08, IETF AIPREF working group, work in progress, not an RFC},
  year = {2026},
  url = {https://datatracker.ietf.org/doc/draft-ietf-aipref-vocab/},
  note = {Revision 08 dated 2026-09-14. Retrieved 2026-10-04}
}

@misc{aipref2026attach,
  author = {Illyes, Gary and Thomson, Martin},
  title = {Associating {AI} Usage Preferences with Content in {HTTP}},
  howpublished = {Internet-Draft draft-ietf-aipref-attach-05, IETF AIPREF working group, work in progress, not an RFC},
  year = {2026},
  url = {https://datatracker.ietf.org/doc/draft-ietf-aipref-attach/},
  note = {Revision 05 dated 2026-08-19. Retrieved 2026-10-04}
}

@misc{rsl2025spec,
  author = {{RSL Technical Steering Committee}},
  title = {Really Simple Licensing ({RSL}) 1.0 Specification},
  howpublished = {RSL-SPEC-1.0, industry specification},
  year = {2025},
  url = {https://rslstandard.org/rsl},
  note = {Published 2025-12-10. Retrieved 2026-10-04}
}

@techreport{tdmrep2024,
  author = {Le Meur, Laurent},
  title = {{TDM} Reservation Protocol ({TDMRep})},
  institution = {W3C Text and Data Mining Reservation Protocol Community Group},
  year = {2024},
  howpublished = {Final Community Group Report, 10 May 2024; not a W3C Recommendation},
  url = {https://www.w3.org/community/reports/tdmrep/CG-FINAL-tdmrep-20240510/},
  note = {Retrieved 2026-10-04}
}

@misc{canel2025robotsai,
  author = {Canel, Fabrice and Madhavan, Krishna},
  title = {Robots Exclusion Protocol Extension to communicate {AI} preferences vocabulary},
  howpublished = {Internet-Draft draft-canel-robots-ai-control-01, individual submission; latest revision 2025-04-04; expired, datatracker state last updated 2025-10-06},
  year = {2025},
  url = {https://datatracker.ietf.org/doc/draft-canel-robots-ai-control/},
  note = {Retrieved 2026-10-04; re-checked 2026-10-10}
}

@misc{romm2025contentsignals,
  author = {Tremante, Michael and Romm, Leah},
  title = {Vocabulary For Expressing Content Signals},
  howpublished = {Internet-Draft draft-romm-aipref-contentsignals-00, individual submission; dated 2025-10-01; expired 2026-04-04 with no successor revision},
  year = {2025},
  url = {https://datatracker.ietf.org/doc/draft-romm-aipref-contentsignals/},
  note = {Defines the three labels only; the Content-Signal line syntax is not in any IETF document. Retrieved 2026-10-04; re-checked 2026-10-10}
}

@misc{deviantart2022noai,
  author = {{DeviantArt}},
  title = {{UPDATE} All Deviations Are Opted Out of {AI} Datasets},
  howpublished = {DeviantArt team journal, 11 November 2022},
  year = {2022},
  url = {https://www.deviantart.com/team/journal/UPDATE-All-Deviations-Are-Opted-Out-of-AI-Datasets-934500371},
  note = {Defines the noai and noimageai meta and header values. Retrieved 2026-10-04}
}

@misc{webbotauth2026draft,
  author = {Meunier, Thibault and Major, Sandor},
  title = {{HTTP} Message Signatures for automated traffic},
  howpublished = {Internet-Draft draft-ietf-webbotauth-httpsig-protocol-00, IETF, work in progress, not an RFC},
  year = {2026},
  url = {https://datatracker.ietf.org/doc/draft-ietf-webbotauth-httpsig-protocol/},
  note = {Dated 2026-09-01. Retrieved 2026-10-04}
}

@misc{microsoft2025nlweb,
  author = {{Microsoft}},
  title = {{NLWeb}: source repository and {REST} {API} documentation},
  howpublished = {GitHub repository nlweb-ai/NLWeb, formerly microsoft/NLWeb},
  year = {2025},
  url = {https://github.com/nlweb-ai/NLWeb},
  note = {Announced 19 May 2025. Retrieved 2026-10-04}
}

@misc{cloudflare2026markdown,
  author = {{Cloudflare}},
  title = {Markdown for Agents},
  howpublished = {Cloudflare developer documentation},
  year = {2026},
  url = {https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/},
  note = {Page last updated 2026-07-13. Retrieved 2026-10-04}
}

@misc{cloudflare2026managedrobots,
  author = {{Cloudflare}},
  title = {Managed robots.txt (robots.txt setting)},
  howpublished = {Cloudflare developer documentation, Bots},
  year = {2026},
  url = {https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/},
  note = {Page last updated 2026-08-03. Retrieved 2026-10-04}
}

@misc{cloudflare2026aioptions,
  author = {{Cloudflare}},
  title = {Block {AI} Bots: Search, Agent and Training presets},
  howpublished = {Cloudflare developer documentation, Bots},
  year = {2026},
  url = {https://developers.cloudflare.com/bots/additional-configurations/block-ai-bots/},
  note = {Page last updated 2026-07-01. Retrieved 2026-10-04}
}

@misc{cloudflare2026botpreferencesync,
  author = {Lee, Jin-Hee},
  title = {Say it once: introducing Bot Preference Sync},
  howpublished = {Cloudflare Blog, 21 August 2026},
  year = {2026},
  url = {https://blog.cloudflare.com/bot-preference-sync/},
  note = {Retrieved 2026-10-04}
}

@misc{cloudflare2026mixeduse,
  author = {Becker, Bryan},
  title = {Have it both ways: stay discoverable in search while disallowing {AI} training},
  howpublished = {Cloudflare Blog, 15 September 2026},
  year = {2026},
  url = {https://blog.cloudflare.com/accountable-mixed-use-ai-crawlers/},
  note = {Retrieved 2026-10-04}
}

@misc{cloudflare2025crawlclick,
  author = {Tomé, João},
  title = {The crawl-to-click gap: Cloudflare data on {AI} bots, training, and referrals},
  howpublished = {Cloudflare Blog, 29 August 2025},
  year = {2025},
  url = {https://blog.cloudflare.com/crawlers-click-ai-bots-training/},
  note = {Retrieved 2026-10-04}
}

@misc{cloudflare2026oneyear,
  author = {Weiss, Arielle and Albertson, Zach and Lanfear, Emily},
  title = {Content Independence Day, one year on: building the business model for the agentic Internet},
  howpublished = {Cloudflare Blog, 1 July 2026},
  year = {2026},
  url = {https://blog.cloudflare.com/agentic-internet-bot-report/},
  note = {Retrieved 2026-10-04}
}

@misc{cloudflare2025perplexity,
  author = {Corral, Gabriel and Singhal, Vaibhav and Mitchell, Brian and Tatoris, Reid},
  title = {Perplexity is using stealth, undeclared crawlers to evade website no-crawl directives},
  howpublished = {Cloudflare Blog, 4 August 2025},
  year = {2025},
  url = {https://blog.cloudflare.com/perplexity-is-using-stealth-undeclared-crawlers-to-evade-website-no-crawl-directives/},
  note = {An allegation by one party; the named company's response was not read for this report. Retrieved 2026-10-04}
}

@misc{openai2026crawlers,
  author = {{OpenAI}},
  title = {Overview of {OpenAI} Crawlers},
  howpublished = {OpenAI developer documentation},
  year = {2026},
  url = {https://developers.openai.com/api/docs/bots},
  note = {Undated page. Retrieved 2026-10-04}
}

@misc{anthropic2026crawler,
  author = {{Anthropic}},
  title = {Does Anthropic crawl data from the web, and how can site owners block the crawler?},
  howpublished = {Claude Help Center article},
  year = {2026},
  url = {https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler},
  note = {Page dated 2026-04-07. Retrieved 2026-10-04}
}

@misc{google2026crawlers,
  author = {{Google}},
  title = {Google's common crawlers},
  howpublished = {Google Search Central documentation},
  year = {2026},
  url = {https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers},
  note = {Page last updated 2026-07-14. Retrieved 2026-10-04}
}

@misc{google2026fetchers,
  author = {{Google}},
  title = {Google User-Triggered Fetchers},
  howpublished = {Google Search Central documentation},
  year = {2026},
  url = {https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers},
  note = {Page last updated 2026-08-19. Retrieved 2026-10-04}
}

@misc{google2025aifeatures,
  author = {{Google}},
  title = {{AI} features and your website},
  howpublished = {Google Search Central documentation},
  year = {2025},
  url = {https://developers.google.com/search/docs/appearance/ai-features},
  note = {Page last updated 2025-12-10. Retrieved 2026-10-04}
}

@misc{google2026searchcontrol,
  author = {Loew, Mrinalini},
  title = {New opportunities, control and insights for website owners},
  howpublished = {Google Keyword blog, 3 June 2026, with an update dated 31 August 2026; and Search Console Help, Search generative AI control},
  year = {2026},
  url = {https://blog.google/products-and-platforms/products/search/new-controls-website-owners/},
  note = {Retrieved 2026-10-04}
}

@misc{meta2026crawlers,
  author = {{Meta}},
  title = {Meta Web Crawlers},
  howpublished = {Meta for Developers documentation},
  year = {2026},
  url = {https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers},
  note = {Undated page. Retrieved 2026-10-04}
}

@misc{amazon2026bot,
  author = {{Amazon}},
  title = {About {AmazonBot}},
  howpublished = {Amazon Developer documentation},
  year = {2026},
  url = {https://developer.amazon.com/amazonbot},
  note = {Undated page. Retrieved 2026-10-04}
}

@misc{perplexity2026crawlers,
  author = {{Perplexity}},
  title = {Perplexity Crawlers},
  howpublished = {Perplexity documentation},
  year = {2026},
  url = {https://docs.perplexity.ai/docs/resources/perplexity-crawlers},
  note = {Undated page. Retrieved 2026-10-04}
}

@misc{mistral2026crawlers,
  author = {{Mistral AI}},
  title = {Mistral crawlers},
  howpublished = {Mistral AI documentation},
  year = {2026},
  url = {https://docs.mistral.ai/robots},
  note = {Undated page. Retrieved 2026-10-04}
}

@misc{apple2026applebot,
  author = {{Apple}},
  title = {About Applebot},
  howpublished = {Apple Support},
  year = {2026},
  url = {https://support.apple.com/en-us/119829},
  note = {Published 2026-09-04. Retrieved 2026-10-04}
}

@misc{commoncrawl2026ccbot,
  author = {{Common Crawl}},
  title = {{CCBot}},
  howpublished = {Common Crawl website},
  year = {2026},
  url = {https://commoncrawl.org/ccbot},
  note = {Undated page. Retrieved 2026-10-04}
}

@misc{airobotstxt2026,
  author = {{ai.robots.txt project}},
  title = {ai.robots.txt: a list of {AI} agents and robots to block},
  howpublished = {Source repository, release v2.0},
  year = {2026},
  url = {https://github.com/ai-robots-txt/ai.robots.txt},
  note = {Release v2.0 dated 2026-10-03, 181 tokens. Retrieved 2026-10-04}
}

@misc{stein2025robots,
  author = {{United States District Court for the Southern District of New York}},
  title = {In re {OpenAI}, Inc. Copyright Infringement Litigation, No. 25-md-3143: Opinion and Order, Dkt. 968 (Stein, J.)},
  howpublished = {Filed opinion, 15 December 2025, RECAP mirror of the PACER original},
  year = {2025},
  url = {https://storage.courtlistener.com/recap/gov.uscourts.nysd.640396/gov.uscourts.nysd.640396.968.0.pdf},
  note = {Retrieved 2026-10-04}
}

@misc{reddit2026serpapi,
  author = {{United States District Court for the Southern District of New York}},
  title = {Reddit, Inc. v. {SerpApi} {LLC} et al., No. 25 Civ. 8736: Opinion and Order, Dkt. 104 (Engelmayer, J.)},
  howpublished = {Filed opinion, 31 July 2026, RECAP mirror of the PACER original},
  year = {2026},
  url = {https://storage.courtlistener.com/recap/gov.uscourts.nysd.651592/gov.uscourts.nysd.651592.104.0.pdf},
  note = {Retrieved 2026-10-04}
}

@misc{amazon2026perplexity,
  author = {{United States Court of Appeals for the Ninth Circuit}},
  title = {Amazon.com Services, {LLC} v. Perplexity {AI}, Inc., No. 26-1444: Opinion (for publication)},
  howpublished = {Filed opinion, 4 August 2026; rehearing en banc denied 10 September 2026},
  year = {2026},
  url = {https://storage.courtlistener.com/recap/gov.uscourts.ca9.9bb73aa1-f12c-4519-875b-262201f0c5d5/gov.uscourts.ca9.9bb73aa1-f12c-4519-875b-262201f0c5d5.61.1.pdf},
  note = {Retrieved 2026-10-04}
}

@misc{bartz2025fairuse,
  author = {{United States District Court for the Northern District of California}},
  title = {Bartz v. Anthropic {PBC}, No. C 24-05417 {WHA}: Order on Fair Use, Dkt. 231 (Alsup, J.)},
  howpublished = {Filed order, 23 June 2025, RECAP mirror of the PACER original},
  year = {2025},
  url = {https://storage.courtlistener.com/recap/gov.uscourts.cand.434709/gov.uscourts.cand.434709.231.0_2.pdf},
  note = {Retrieved 2026-10-04}
}

@misc{kadrey2025order,
  author = {{United States District Court for the Northern District of California}},
  title = {Kadrey v. Meta Platforms, Inc., No. 23-cv-03417-{VC}: Order, Dkt. 598 (Chhabria, J.)},
  howpublished = {Filed order, 25 June 2025, RECAP mirror of the PACER original},
  year = {2025},
  url = {https://storage.courtlistener.com/recap/gov.uscourts.cand.415175/gov.uscourts.cand.415175.598.0_1.pdf},
  note = {Retrieved 2026-10-04}
}

@misc{ross2026ca3,
  author = {{United States Court of Appeals for the Third Circuit}},
  title = {Thomson Reuters Enterprise Centre {GmbH} v. Ross Intelligence Inc., No. 25-2153: Opinion (precedential)},
  howpublished = {Filed opinion, 29 September 2026, unsealed 30 September 2026},
  year = {2026},
  url = {https://storage.courtlistener.com/recap/gov.uscourts.ca3.125346/gov.uscourts.ca3.125346.214.0.pdf},
  note = {Retrieved 2026-10-04}
}

@techreport{usco2025part3,
  author = {{United States Copyright Office}},
  title = {Copyright and Artificial Intelligence, Part 3: Generative {AI} Training (Pre-Publication Version)},
  institution = {United States Copyright Office},
  year = {2025},
  url = {https://www.copyright.gov/ai/Copyright-and-Artificial-Intelligence-Part-3-Generative-AI-Training-Report-Pre-Publication-Version.pdf},
  note = {Report of the Register of Copyrights, May 2025. Retrieved 2026-10-04}
}

@misc{eu2019dsm,
  author = {{European Parliament and Council of the European Union}},
  title = {Directive ({EU}) 2019/790 of 17 April 2019 on copyright and related rights in the Digital Single Market, Article 4},
  howpublished = {Official Journal of the European Union, L 130; text read in the official UK reproduction},
  year = {2019},
  url = {https://www.legislation.gov.uk/eudr/2019/790/article/4},
  note = {Retrieved 2026-10-04}
}

@techreport{eu2025gpaicode,
  author = {{European Commission, AI Office}},
  title = {Code of Practice for General-Purpose {AI} Models: Copyright Chapter},
  institution = {European Commission},
  year = {2025},
  url = {https://digital-strategy.ec.europa.eu/en/policies/contents-code-gpai},
  note = {Published 10 July 2025; the Commission page lists the signatories. Retrieved 2026-10-04}
}

@techreport{euipo2025genai,
  author = {{European Union Intellectual Property Office}},
  title = {The Development of Generative Artificial Intelligence from a Copyright Perspective},
  institution = {European Union Intellectual Property Office},
  year = {2025},
  url = {https://www.euipo.europa.eu/en/publications/genai-from-a-copyright-perspective-2025},
  note = {May 2025, ISBN 978-92-9156-369-2, DOI 10.2814/3893780. Retrieved 2026-10-04}
}

@misc{ec2025optoutconsult,
  author = {{European Commission}},
  title = {Consultation on protocols for reserving rights from text and data mining under the {AI} Act and the {GPAI} Code of Practice},
  howpublished = {Consultation page, Shaping Europe's digital future},
  year = {2025},
  url = {https://digital-strategy.ec.europa.eu/en/consultations/commission-launches-consultation-protocols-reserving-rights-text-and-data-mining-under-ai-act-and},
  note = {Consultation open 1 December 2025 to 23 January 2026; page last updated 16 December 2025. Retrieved 2026-10-04}
}

@techreport{uk2026copyreport,
  author = {{UK Department for Science, Innovation and Technology and Intellectual Property Office}},
  title = {Report on Copyright and Artificial Intelligence},
  institution = {UK Government},
  year = {2026},
  url = {https://www.gov.uk/government/publications/report-and-impact-assessment-on-copyright-and-artificial-intelligence/report-on-copyright-and-artificial-intelligence},
  note = {Policy paper laid under the Data (Use and Access) Act 2025, 18 March 2026. Retrieved 2026-10-04}
}

@techreport{cma2026publisher,
  author = {{UK Competition and Markets Authority}},
  title = {Google Search: Publisher Conduct Requirement, notice under section 19(1) of the Digital Markets, Competition and Consumers Act 2024, and final decision},
  institution = {Competition and Markets Authority},
  year = {2026},
  url = {https://www.gov.uk/find-digital-markets-measures/google-search-publisher-conduct-requirement},
  note = {Imposed 3 June 2026; in force from 3 December 2026. Retrieved 2026-10-04}
}

@misc{pew2025aisummary,
  author = {Chapekis, Athena and Lieb, Anna},
  title = {Google users are less likely to click on links when an {AI} summary appears in the results},
  howpublished = {Pew Research Center short read, 22 July 2025},
  year = {2025},
  url = {https://www.pewresearch.org/short-reads/2025/07/22/google-users-are-less-likely-to-click-on-links-when-an-ai-summary-appears-in-the-results/},
  note = {Retrieved 2026-10-04}
}

@techreport{risj2026dnr,
  author = {Egan, Jim},
  title = {Overview and key findings of the 2026 Digital News Report},
  institution = {Reuters Institute for the Study of Journalism, University of Oxford},
  year = {2026},
  url = {https://reutersinstitute.politics.ox.ac.uk/digital-news-report/2026/dnr-executive-summary},
  note = {Published 16 June 2026. Retrieved 2026-10-04}
}

@misc{dcn2025traffic,
  author = {Kint, Jason},
  title = {Facts: Google's push to {AI} hurts publisher traffic},
  howpublished = {Digital Content Next, 14 August 2025},
  year = {2025},
  url = {https://digitalcontentnext.org/blog/2025/08/14/facts-googles-push-to-ai-hurts-publisher-traffic/},
  note = {A publishers' trade body. Retrieved 2026-10-04}
}

@misc{google2025reid,
  author = {Reid, Liz},
  title = {{AI} in Search is driving more queries and higher quality clicks},
  howpublished = {Google Keyword blog, 6 August 2025},
  year = {2025},
  url = {https://blog.google/products/search/ai-search-driving-more-queries-higher-quality-clicks/},
  note = {Retrieved 2026-10-04}
}

@misc{ap2023openai,
  author = {{The Associated Press}},
  title = {{AP} and {OpenAI} agree to share select news content and technology in new collaboration},
  howpublished = {Press release, 13 July 2023},
  year = {2023},
  url = {https://www.ap.org/media-center/press-releases/2023/ap-open-ai-agree-to-share-select-news-content-and-technology-in-new-collaboration/},
  note = {Retrieved 2026-10-04}
}

@misc{reddit2024google,
  author = {{Reddit}},
  title = {Expanding our Partnership with Google},
  howpublished = {Reddit Upvoted blog, 22 February 2024},
  year = {2024},
  url = {https://www.redditinc.com/blog/reddit-and-google-expand-partnership},
  note = {Retrieved 2026-10-04}
}

@misc{microsoft2026pcm,
  author = {Frank, Tim},
  title = {Building Toward a Sustainable Content Economy for the Agentic Web},
  howpublished = {Microsoft Advertising Blog, 3 February 2026},
  year = {2026},
  url = {https://about.ads.microsoft.com/en/blog/post/february-2026/building-toward-a-sustainable-content-economy-for-the-agentic-web},
  note = {Retrieved 2026-10-04}
}

@misc{wikimedia2025crawlers,
  author = {Mueller, Birgit and Danis, Chris and Lavagetto, Giuseppe},
  title = {How crawlers impact the operations of the Wikimedia projects},
  howpublished = {Diff, Wikimedia Foundation, 1 April 2025},
  year = {2025},
  url = {https://diff.wikimedia.org/2025/04/01/how-crawlers-impact-the-operations-of-the-wikimedia-projects/},
  note = {Retrieved 2026-10-04}
}

@misc{readthedocs2024crawlers,
  author = {Holscher, Eric},
  title = {{AI} crawlers need to be more respectful},
  howpublished = {Read the Docs blog, 25 July 2024},
  year = {2024},
  url = {https://about.readthedocs.com/blog/2024/07/ai-crawlers-abuse/},
  note = {Retrieved 2026-10-04}
}

@misc{anubis2025,
  author = {Iaso, Xe},
  title = {Block {AI} scrapers with Anubis},
  howpublished = {Personal blog, 19 January 2025},
  year = {2025},
  url = {https://xeiaso.net/blog/2025/anubis/},
  note = {Retrieved 2026-10-04}
}

@misc{google2025ap2,
  author = {Parikh, Stavan and Surapaneni, Rao},
  title = {Powering {AI} commerce with the new Agent Payments Protocol ({AP2})},
  howpublished = {Google Cloud Blog, 16 September 2025},
  year = {2025},
  url = {https://cloud.google.com/blog/products/ai-machine-learning/announcing-agents-to-payments-ap2-protocol},
  note = {Retrieved 2026-10-04}
}

@misc{stripe2025acp,
  author = {Weinstein, Jeff and Kaliski, Steve},
  title = {Developing an open standard for agentic commerce},
  howpublished = {Stripe Blog, 29 September 2025},
  year = {2025},
  url = {https://stripe.com/blog/developing-an-open-standard-for-agentic-commerce},
  note = {Retrieved 2026-10-04}
}

@misc{adobe2025semrush,
  author = {{Adobe}},
  title = {Adobe to Acquire Semrush},
  howpublished = {Press release, 19 November 2025},
  year = {2025},
  url = {https://news.adobe.com/news/2025/11/adobe-to-acquire-semrush},
  note = {Retrieved 2026-10-04}
}