% GenAI Security Literature Review -- 95 entries
% https://github.com/emmanuelgjr/GenAI-Security-Literature-Review (MIT License), generated 2026-09-15
% Entries with reviewed=false in literature.json were added automatically and are not yet curated.

@article{greshake2023not,
  title = {{Not What You've Signed Up For: Compromising Real-World LLM-Integrated Applications with Indirect Prompt Injection}},
  author = {Kai Greshake and Sahar Abdelnabi and Shailesh Mishra and Christoph Endres and Thorsten Holz and Mario Fritz},
  year = {2023},
  month = may,
  journal = {AISec 2023},
  eprint = {2302.12173},
  archivePrefix = {arXiv},
  doi = {10.1145/3605764.3623985},
  url = {https://arxiv.org/abs/2302.12173}
}

@inproceedings{schulhoff2023ignore,
  title = {{Ignore This Title and HackAPrompt: Exposing Systemic Weaknesses of LLMs through a Global Scale Prompt Hacking Competition}},
  author = {Sander Schulhoff and Jeremy Pinto and Anaum Khan and Louis-Francois Bouchard and Chenglei Si and Svetlina Anati and Valen Tagliabue and Anson Liu Kost and Christopher Carnahan and Jordan Boyd-Graber},
  year = {2023},
  month = nov,
  booktitle = {EMNLP 2023},
  eprint = {2311.16119},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2311.16119}
}

@inproceedings{wei2024jailbroken,
  title = {{Jailbroken: How Does LLM Safety Training Fail?}},
  author = {Alexander Wei and Nika Haghtalab and Jacob Steinhardt},
  year = {2024},
  month = jan,
  booktitle = {NeurIPS 2023},
  eprint = {2307.02483},
  archivePrefix = {arXiv},
  doi = {10.52202/075280-3508},
  url = {https://arxiv.org/abs/2307.02483}
}

@misc{zou2023universal,
  title = {{Universal and Transferable Adversarial Attacks on Aligned Language Models}},
  author = {Andy Zou and Zifan Wang and Nicholas Carlini and Milad Nasr and J. Zico Kolter and Matt Fredrikson},
  year = {2023},
  month = jul,
  eprint = {2307.15043},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2307.15043}
}

@inproceedings{wan2023poisoning,
  title = {{Poisoning Language Models During Instruction Tuning}},
  author = {Alexander Wan and Eric Wallace and Sheng Shen and Dan Klein},
  year = {2023},
  month = may,
  booktitle = {ICML 2023},
  eprint = {2305.00944},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2305.00944}
}

@misc{zou2024poisonedrag,
  title = {{PoisonedRAG: Knowledge Poisoning Attacks to Retrieval-Augmented Generation of Large Language Models}},
  author = {Wei Zou and Runpeng Geng and Binghui Wang and Jinyuan Jia},
  year = {2024},
  month = feb,
  eprint = {2402.07867},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.07867}
}

@inproceedings{carlini2021extracting,
  title = {{Extracting Training Data from Large Language Models}},
  author = {Nicholas Carlini and Florian Tramer and Eric Wallace and Matthew Jagielski and Ariel Herbert-Voss and Katherine Lee and Adam Roberts and Tom Brown and Dawn Song and Ulfar Erlingsson and Alina Oprea and Colin Raffel},
  year = {2021},
  booktitle = {USENIX Security 2021},
  eprint = {2012.07805},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2012.07805}
}

@misc{nasr2023scalable,
  title = {{Scalable Extraction of Training Data from (Production) Language Models}},
  author = {Milad Nasr and Nicholas Carlini and Jonathan Hayase and Matthew Jagielski and A. Feder Cooper and Daphne Ippolito and Christopher A. Choquette-Choo and Eric Wallace and Florian Tramer and Katherine Lee},
  year = {2023},
  month = nov,
  eprint = {2311.17035},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2311.17035}
}

@inproceedings{carlini2024stealing,
  title = {{Stealing Part of a Production Language Model}},
  author = {Nicholas Carlini and Daniel Paleka and Krishnamurthy Dj Dvijotham and Thomas Steinke and Jonathan Hayase and A. Feder Cooper and Katherine Lee and Matthew Jagielski and Milad Nasr and Arthur Conmy and Eric Wallace and David Rolnick and Florian Tramer},
  year = {2024},
  month = mar,
  booktitle = {ICML 2024},
  eprint = {2403.06634},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2403.06634}
}

@misc{wallace2024instruction,
  title = {{The Instruction Hierarchy: Training LLMs to Prioritize Privileged Instructions}},
  author = {Eric Wallace and Kai Xiao and Reimar Leike and Lilian Weng and Johannes Heidecke and Alex Beutel},
  year = {2024},
  month = apr,
  eprint = {2404.13208},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2404.13208}
}

@misc{inan2023llama,
  title = {{Llama Guard: LLM-based Input-Output Safeguard for Human-AI Conversations}},
  author = {Hakan Inan and Kartikeya Upasani and Jianfeng Chi and Rashi Rungta and Krithika Iyer and Yuning Mao and Michael Tontchev and Qing Hu and Brian Fuller and Davide Testuggine and Madian Khabsa},
  year = {2023},
  month = dec,
  eprint = {2312.06674},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2312.06674}
}

@inproceedings{rebedea2023nemo,
  title = {{NeMo Guardrails: A Toolkit for Controllable and Safe LLM Applications with Programmable Rails}},
  author = {Traian Rebedea and Razvan Dinu and Makesh Narsimhan Sreedhar and Christopher Parisien and Jonathan Cohen},
  year = {2023},
  month = oct,
  booktitle = {EMNLP 2023 Demo},
  eprint = {2310.10501},
  archivePrefix = {arXiv},
  doi = {10.18653/v1/2023.emnlp-demo.40},
  url = {https://arxiv.org/abs/2310.10501}
}

@article{yao2024survey,
  title = {{A Survey on Large Language Model (LLM) Security and Privacy: The Good, The Bad, and The Ugly}},
  author = {Yifan Yao and Jinhao Duan and Kaidi Xu and Yuanfang Cai and Zhibo Sun and Yue Zhang},
  year = {2024},
  month = mar,
  journal = {High-Confidence Computing},
  eprint = {2312.02003},
  archivePrefix = {arXiv},
  doi = {10.1016/j.hcc.2024.100211},
  url = {https://arxiv.org/abs/2312.02003}
}

@misc{schwinn2024adversarial,
  title = {{Adversarial Attacks and Defenses in Large Language Models: Old and New Threats}},
  author = {Leo Schwinn and David Dobre and Stephan Gunnemann and Gauthier Gidel},
  year = {2024},
  month = jan,
  eprint = {2310.19737},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.19737}
}

@misc{esmradi2024comprehensive,
  title = {{A Comprehensive Survey of Attack Techniques, Implementation, and Mitigation Strategies in Large Language Models}},
  author = {Aysan Esmradi and Daniel Wankit Yip and Chun Fai Chan},
  year = {2024},
  month = dec,
  eprint = {2312.10982},
  archivePrefix = {arXiv},
  doi = {10.1007/978-981-97-1274-8_6},
  url = {https://arxiv.org/abs/2312.10982}
}

@misc{debenedetti2024agentdojo,
  title = {{AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents}},
  author = {Edoardo Debenedetti and Jie Zhang and Mislav Balunovic and Luca Beurer-Kellner and Marc Fischer and Florian Tramer},
  year = {2024},
  month = jun,
  eprint = {2406.13352},
  archivePrefix = {arXiv},
  doi = {10.52202/079017-2636},
  url = {https://arxiv.org/abs/2406.13352}
}

@inproceedings{zhan2024injecagent,
  title = {{InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated LLM Agents}},
  author = {Qiusi Zhan and Zhixiang Liang and Zifan Ying and Daniel Kang},
  year = {2024},
  month = mar,
  booktitle = {ACL 2024 Findings},
  eprint = {2403.02691},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2403.02691}
}

@inproceedings{shen2023do,
  title = {{Do Anything Now: Characterizing and Evaluating In-The-Wild Jailbreak Prompts on Large Language Models}},
  author = {Xinyue Shen and Zeyuan Chen and Michael Backes and Yun Shen and Yang Zhang},
  year = {2023},
  month = aug,
  booktitle = {CCS 2024},
  eprint = {2308.03825},
  archivePrefix = {arXiv},
  doi = {10.1145/3658644.3670388},
  url = {https://arxiv.org/abs/2308.03825}
}

@inproceedings{li2023multistep,
  title = {{Multi-step Jailbreaking Privacy Attacks on ChatGPT}},
  author = {Haoran Li and Dadi Guo and Wei Fan and Mingshi Xu and Jie Huang and Fanpu Meng and Yangqiu Song},
  year = {2023},
  month = apr,
  booktitle = {EMNLP 2023 Findings},
  eprint = {2304.05197},
  archivePrefix = {arXiv},
  doi = {10.18653/v1/2023.findings-emnlp.272},
  url = {https://arxiv.org/abs/2304.05197}
}

@inproceedings{kirchenbauer2023text,
  title = {{A Text Watermark for Large Language Models}},
  author = {John Kirchenbauer and Jonas Geiping and Yuxin Wen and Jonathan Katz and Ian Miers and Tom Goldstein},
  year = {2023},
  month = jun,
  booktitle = {ICML 2023},
  eprint = {2301.10226},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2301.10226}
}

@inproceedings{mireshghallah2024can,
  title = {{Can LLMs Keep a Secret? Testing Privacy Implications of Language Models via Contextual Integrity Theory}},
  author = {Niloofar Mireshghallah and Hyunwoo Kim and Xuhui Zhou and Yulia Tsvetkov and Maarten Sap and Reza Shokri and Yejin Choi},
  year = {2024},
  month = jan,
  booktitle = {ICLR 2024},
  eprint = {2310.17884},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.17884}
}

@misc{liu2024machine,
  title = {{Machine Unlearning in Generative AI: A Survey}},
  author = {Zheyuan Liu and Guangyao Dou and Zhaoxuan Tan and Yijun Tian and Meng Jiang},
  year = {2024},
  eprint = {2407.20516},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2407.20516}
}

@misc{ganguli2022red,
  title = {{Red Teaming Language Models to Reduce Harms: Methods, Scaling Behaviors, and Lessons Learned}},
  author = {Deep Ganguli and Liane Lovitt and Jackson Kernion and Amanda Askell and Yuntao Bai and Saurav Kadavath and Ben Mann and Ethan Perez and Nicholas Schiefer and Kamal Ndousse and Andy Jones and Sam Bowman and Anna Chen and Tom Conerly and Nova DasSarma and Dawn Drain and Nelson Elhage and Sheer El-Showk and Stanislav Fort and Zac Hatfield-Dodds and Tom Henighan and Danny Hernandez and Tristan Hume and Josh Jacobson and Scott Johnston and Shauna Kravec and Catherine Olsson and Sam Ringer and Eli Tyre and Jared Kaplan and Chris Olah and Sam McCandlish and Dario Amodei},
  year = {2022},
  month = aug,
  eprint = {2209.07858},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2209.07858}
}

@misc{derczynski2024garak,
  title = {{Garak: A Framework for Security Probing Large Language Models}},
  author = {Leon Derczynski and Erick Galinkin and Jeffrey Martin and Subho Majumdar and Nanna Inie},
  year = {2024},
  month = jun,
  eprint = {2406.11036},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2406.11036}
}

@inproceedings{yuan2024rjudge,
  title = {{R-Judge: Benchmarking Safety Risk Awareness for LLM Agents}},
  author = {Tongxin Yuan and Zhiwei He and Lingzhong Dong and Yiming Wang and Ruijie Zhao and Tian Xia and Lizhen Xu and Binglin Zhou and Fangqi Li and Zhuosheng Zhang and Rui Wang and Gongshen Liu},
  year = {2024},
  month = jan,
  booktitle = {EMNLP 2024},
  eprint = {2401.10019},
  archivePrefix = {arXiv},
  doi = {10.18653/v1/2024.findings-emnlp.79},
  url = {https://arxiv.org/abs/2401.10019}
}

@techreport{wilson2025owasp,
  title = {{OWASP Top 10 for Large Language Model Applications}},
  author = {Steve Wilson and {OWASP LLM AI Security Team}},
  year = {2025},
  institution = {OWASP Foundation},
  url = {https://owasp.org/www-project-top-10-for-large-language-model-applications/}
}

@techreport{national2023nist,
  title = {{NIST Artificial Intelligence Risk Management Framework (AI RMF 1.0)}},
  author = {{National Institute of Standards and Technology}},
  year = {2023},
  month = jan,
  institution = {NIST},
  doi = {10.6028/NIST.AI.100-1},
  url = {https://www.nist.gov/itl/ai-risk-management-framework}
}

@techreport{mitre2024mitre,
  title = {{MITRE ATLAS: Adversarial Threat Landscape for AI Systems}},
  author = {{MITRE Corporation}},
  year = {2024},
  institution = {MITRE},
  url = {https://atlas.mitre.org/}
}

@techreport{veer2024owasp,
  title = {{OWASP AI Security and Privacy Guide}},
  author = {Rob van der Veer and {OWASP AI Exchange Team}},
  year = {2024},
  institution = {OWASP Foundation},
  url = {https://owasp.org/www-project-ai-security-and-privacy-guide/}
}

@techreport{anthropic2024anthropic,
  title = {{Anthropic: Many-shot Jailbreaking}},
  author = {{Anthropic}},
  year = {2024},
  month = apr,
  institution = {Anthropic Research Blog},
  url = {https://www.anthropic.com/research/many-shot-jailbreaking}
}

@misc{bai2022constitutional,
  title = {{Constitutional AI: Harmlessness from AI Feedback}},
  author = {Yuntao Bai and Saurav Kadavath and Sandipan Kundu and Amanda Askell and Jackson Kernion and Andy Jones and Anna Chen and Anna Goldie and Azalia Mirhoseini and Cameron McKinnon and Carol Chen and Catherine Olsson and Christopher Olah and Danny Hernandez and Dawn Drain and Deep Ganguli and Dustin Li and Eli Tyre and Ethan Perez and Jamie Kerr and Jared Kaplan and Jeffrey Ladish and Joshua Landau and Kamal Ndousse and Kamile Lukosiute and Liane Lovitt and Michael Sellitto and Nelson Elhage and Nicholas Schiefer and Noemi Mercado and Nova DasSarma and Robert Lasenby and Robin Larson and Sam Ringer and Scott Johnston and Shauna Kravec and Sheer El Showk and Stanislav Fort and Tamera Lanham and Timothy Telleen-Lawton and Tom Brown and Tom Henighan and Tristan Hume and Sam McCandlish and Jared Kaplan and Dario Amodei and Chris Olah},
  year = {2022},
  month = dec,
  eprint = {2212.08073},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2212.08073}
}

@article{liu2024prompt,
  title = {{Prompt Injection Attack Against LLM-Integrated Applications}},
  author = {Yi Liu and Gelei Deng and Yuekang Li and Kailong Wang and Tianwei Zhang and Yepang Liu and Haoyu Wang and Yan Zheng and Yang Liu},
  year = {2024},
  month = jun,
  journal = {ACM Computing Surveys},
  eprint = {2306.05499},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2306.05499}
}

@misc{protect2023rebuff,
  title = {{Rebuff: Self-Hardening Prompt Injection Detector}},
  author = {{Protect AI}},
  year = {2023},
  url = {https://github.com/protectai/rebuff}
}

@misc{microsoft2024pyrit,
  title = {{PyRIT: Python Risk Identification Toolkit for Generative AI}},
  author = {{Microsoft AI Red Team}},
  year = {2024},
  url = {https://github.com/Azure/PyRIT}
}

@misc{hubinger2024sleeper,
  title = {{Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training}},
  author = {Evan Hubinger and Carson Denison and Jesse Mu and Mike Lambert and Meg Tong and Monte MacDiarmid and Tamera Lanham and Daniel M. Ziegler and Tim Maxwell and Newton Cheng and Adam Jermyn and Amanda Askell and Ansh Radhakrishnan and Cem Anil and David Duvenaud and Deep Ganguli and Fazl Barez and Jack Clark and Kamal Ndousse and Kshitij Sachan and Michael Sellitto and Mrinank Sharma and Nova DasSarma and Roger Grosse and Shauna Kravec and Yuntao Bai and Jared Kaplan and Dario Amodei and Sam McCandlish and Ethan Perez},
  year = {2024},
  month = jan,
  eprint = {2401.05566},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2401.05566}
}

@inproceedings{toyer2024tensor,
  title = {{Tensor Trust: Interpretable Prompt Injection Attacks from an Online Game}},
  author = {Sam Toyer and Olivia Watkins and Ethan Adrian Mendes and Justin Svegliato and Luke Bailey and Tiffany Wang and Isaac Ong and Karim Elmaaroufi and Pieter Abbeel and Trevor Darrell and Alan Ritter and Stuart Russell},
  year = {2024},
  month = jan,
  booktitle = {ICLR 2024},
  eprint = {2311.01011},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2311.01011}
}

@techreport{owasp2024owaspa,
  title = {{OWASP LLM AI Security \& Governance Checklist}},
  author = {{OWASP Foundation} and Sandy Dunn and Jackie McGuire},
  year = {2024},
  institution = {OWASP GenAI Security Project},
  url = {https://genai.owasp.org/resource/llm-applications-cybersecurity-and-governance-checklist-english/}
}

@article{woisetschlager2024federated,
  title = {{Federated Fine-Tuning of LLMs on the Very Edge: The Good, the Bad, the Ugly}},
  author = {Herbert Woisetschläger and Alexander Isenko and Shiqiang Wang and Ruben Mayer and Hans-Arno Jacobsen},
  year = {2024},
  journal = {DEEM@SIGMOD 2024},
  eprint = {2310.03150},
  archivePrefix = {arXiv},
  doi = {10.1145/3650203.3663331},
  url = {https://arxiv.org/abs/2310.03150}
}

@inproceedings{xue2024trojllm,
  title = {{TrojLLM: A Black-box Trojan Prompt Attack on Large Language Models}},
  author = {Jiaqi Xue and Mengxin Zheng and Ting Hua and Yilin Shen and Yepeng Liu and Ladislau Boloni and Qian Lou},
  year = {2024},
  month = may,
  booktitle = {NeurIPS 2023},
  eprint = {2306.06815},
  archivePrefix = {arXiv},
  doi = {10.52202/075280-2866},
  url = {https://arxiv.org/abs/2306.06815}
}

@misc{lermen2023lora,
  title = {{LoRA Fine-Tuning Efficiently Undoes Safety Training in Llama 2-Chat}},
  author = {Simon Lermen and Charlie Rogers-Smith and Jeffrey Ladish},
  year = {2023},
  month = oct,
  eprint = {2310.20624},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.20624}
}

@inproceedings{wei2024assessing,
  title = {{Assessing the Brittleness of Safety Alignment via Pruning and Low-Rank Modifications}},
  author = {Boyi Wei and Kaixuan Huang and Yangsibo Huang and Tinghao Xie and Xiangyu Qi and Mengzhou Xia and Prateek Mittal and Mengdi Wang and Peter Henderson},
  year = {2024},
  booktitle = {ICML 2024},
  eprint = {2402.05162},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.05162}
}

@misc{roychowdhury2024confusedpilot,
  title = {{ConfusedPilot: Confused Deputy Risks in RAG-based LLMs}},
  author = {Ayush RoyChowdhury and Mulong Luo and Prateek Sahu and Sarbartha Banerjee and Mohit Tiwari},
  year = {2024},
  eprint = {2408.04870},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2408.04870}
}

@inproceedings{schick2023toolformer,
  title = {{Toolformer: Language Models Can Teach Themselves to Use Tools}},
  author = {Timo Schick and Jane Dwivedi-Yu and Roberto Dessi and Roberta Raileanu and Maria Lomeli and Eric Hambro and Luke Zettlemoyer and Nicola Cancedda and Thomas Scialom},
  year = {2023},
  month = feb,
  booktitle = {NeurIPS 2023},
  eprint = {2302.04761},
  archivePrefix = {arXiv},
  doi = {10.52202/075280-2997},
  url = {https://arxiv.org/abs/2302.04761}
}

@article{barrett2023identifying,
  title = {{Identifying and Mitigating the Security Risks of Generative AI}},
  author = {Clark Barrett and Brad Boyd and Elie Burzstein and Nicholas Carlini and Brad Chen and Jihye Choi and Amrita Roy Chowdhury and Mihai Christodorescu and Anupam Datta and Soheil Feizi and Kathleen Fisher and Tatsunori Hashimoto and Dan Hendrycks and Somesh Jha and Daniel Kang and Florian Kerschbaum and Eric Mitchell and John Mitchell and Zulfikar Ramzan and Khawaja Shams and Dawn Song and Ankur Taly and Diyi Yang},
  year = {2023},
  month = dec,
  journal = {Foundations and Trends in Privacy and Security},
  eprint = {2308.14840},
  archivePrefix = {arXiv},
  doi = {10.1561/3300000041},
  url = {https://arxiv.org/abs/2308.14840}
}

@techreport{ward2024ai,
  title = {{The AI Security Pyramid of Pain}},
  author = {Chris M. Ward and Josh Harguess and Julia Tao and Daniel Christman and Paul Spicer and Mike Tan},
  year = {2024},
  institution = {Proc. SPIE 13054, Assurance and Security for AI-enabled Systems},
  eprint = {2402.11082},
  archivePrefix = {arXiv},
  doi = {10.1117/12.3025025},
  url = {https://arxiv.org/abs/2402.11082}
}

@inproceedings{liu2024formalizing,
  title = {{Formalizing and Benchmarking Prompt Injection Attacks and Defenses}},
  author = {Yupei Liu and Yuqi Jia and Runpeng Geng and Jinyuan Jia and Neil Zhenqiang Gong},
  year = {2024},
  booktitle = {USENIX Security 2024},
  eprint = {2310.12815},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.12815}
}

@techreport{vassilev2024adversarial,
  title = {{Adversarial Machine Learning: A Taxonomy and Terminology of Attacks and Mitigations (NIST AI 100-2e2025)}},
  author = {Apostol Vassilev and Alina Oprea and Alie Fordyce and Hyrum Anderson},
  year = {2024},
  month = jan,
  institution = {NIST},
  url = {https://csrc.nist.gov/pubs/ai/100/2/e2025/final}
}

@misc{fang2024llma,
  title = {{LLM Agents Can Autonomously Hack Websites}},
  author = {Richard Fang and Rohan Bindu and Akul Gupta and Qiusi Zhan and Daniel Kang},
  year = {2024},
  month = feb,
  eprint = {2402.06664},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.06664}
}

@misc{wang2024pandoras,
  title = {{Pandora's White-Box: Precise Training Data Detection and Extraction in Large Language Models}},
  author = {Jeffrey G. Wang and Jason Wang and Marvin Li and Seth Neel},
  year = {2024},
  eprint = {2402.17012},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.17012}
}

@techreport{owasp2025owasp,
  title = {{OWASP Top 10 for Agentic Applications 2026}},
  author = {{OWASP GenAI Security Project}},
  year = {2025},
  month = dec,
  institution = {OWASP GenAI Security Project},
  url = {https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/}
}

@inproceedings{pedro2024from,
  title = {{From Prompt Injections to SQL Injection Attacks: How Protected is Your LLM-Integrated Web Application?}},
  author = {Rodrigo Pedro and Daniel Castro and Paolo Molina and Nuno Santos},
  year = {2024},
  month = aug,
  booktitle = {USENIX Security 2024},
  eprint = {2308.01990},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2308.01990}
}

@techreport{anthropic2024modela,
  title = {{Model Context Protocol (MCP): Security Best Practices}},
  author = {{Anthropic}},
  year = {2024},
  institution = {Anthropic Documentation},
  url = {https://modelcontextprotocol.io/docs/tutorials/security/security_best_practices}
}

@misc{yi2024benchmarking,
  title = {{Benchmarking and Defending Against Indirect Prompt Injection Attacks on Large Language Models}},
  author = {Jingwei Yi and Yueqi Xie and Bin Zhu and Keegan Hines and Emre Kiciman and Guangzhong Sun and Xing Xie and Fangzhao Wu},
  year = {2024},
  month = jan,
  eprint = {2312.14197},
  archivePrefix = {arXiv},
  doi = {10.1145/3690624.3709179},
  url = {https://arxiv.org/abs/2312.14197}
}

@techreport{mcgraw2024architectural,
  title = {{An Architectural Risk Analysis of Large Language Models: Applied Machine Learning Security}},
  author = {Gary McGraw and Harold Figueroa and Katie McMahon and Richie Bonett},
  year = {2024},
  month = jan,
  institution = {Berryville Institute of Machine Learning (BIML)},
  url = {https://berryvilleiml.com/docs/BIML-LLM24.pdf}
}

@misc{mazeika2024harmbench,
  title = {{HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal}},
  author = {Mantas Mazeika and Long Phan and Xuwang Yin and Andy Zou and Zifan Wang and Norman Mu and Elham Sakhaee and Nathaniel Li and Steven Basart and Bo Li and David Forsyth and Dan Hendrycks},
  year = {2024},
  month = feb,
  eprint = {2402.04249},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.04249}
}

@techreport{international2023isoiec,
  title = {{ISO/IEC 42001:2023 - Artificial Intelligence Management System}},
  author = {{International Organization for Standardization}},
  year = {2023},
  month = dec,
  institution = {ISO},
  url = {https://www.iso.org/standard/81230.html}
}

@inproceedings{liu2024autodan,
  title = {{AutoDAN: Generating Stealthy Jailbreak Prompts on Aligned Large Language Models}},
  author = {Xiaogeng Liu and Nan Xu and Muhao Chen and Chaowei Xiao},
  year = {2024},
  booktitle = {ICLR 2024},
  eprint = {2310.04451},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.04451}
}

@inproceedings{mehrotra2024tree,
  title = {{Tree of Attacks: Jailbreaking Black-Box LLMs with Auto-Generated Subtrees}},
  author = {Anay Mehrotra and Manolis Zampetakis and Paul Kassianik and Blaine Nelson and Hyrum Anderson and Yaron Singer and Amin Karbasi},
  year = {2024},
  booktitle = {NeurIPS 2024},
  eprint = {2312.02119},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2312.02119}
}

@inproceedings{yuan2024gpt4,
  title = {{GPT-4 Is Too Smart To Be Safe: Stealthy Chat with LLMs via Cipher}},
  author = {Youliang Yuan and Wenxiang Jiao and Wenxuan Wang and Jen-tse Huang and Pinjia He and Shuming Shi and Zhaopeng Tu},
  year = {2024},
  booktitle = {ICLR 2024},
  eprint = {2308.06463},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2308.06463}
}

@inproceedings{wang2024decodingtrust,
  title = {{DecodingTrust: A Comprehensive Assessment of Trustworthiness in GPT Models}},
  author = {Boxin Wang and Weixin Chen and Hengzhi Pei and Chulin Xie and Mintong Kang and Chenhui Zhang and Chejian Xu and Zidi Xiong and Ritik Dutta and Rylan Schaeffer},
  year = {2024},
  booktitle = {NeurIPS 2023},
  eprint = {2306.11698},
  archivePrefix = {arXiv},
  doi = {10.52202/075280-1361},
  url = {https://arxiv.org/abs/2306.11698}
}

@misc{bhatt2024purple,
  title = {{Purple Llama CyberSecEval: A Secure Coding Benchmark for Language Models}},
  author = {Manish Bhatt and Sahana Chennabasappa and Cyrus Nikolaidis and Shengye Wan and Ivan Evtimov},
  year = {2024},
  eprint = {2312.04724},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2312.04724}
}

@inproceedings{xiang2024badchain,
  title = {{BadChain: Backdoor Chain-of-Thought Prompting for Large Language Models}},
  author = {Zhen Xiang and Fengqing Jiang and Zidi Xiong and Bhaskar Ramasubramanian and Radha Poovendran and Bo Li},
  year = {2024},
  booktitle = {NeurIPS 2024},
  eprint = {2401.12242},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2401.12242}
}

@article{chao2025jailbreaking,
  title = {{Jailbreaking Black Box Large Language Models in Twenty Queries}},
  author = {Patrick Chao and Alexander Robey and Edgar Dobriban and Hamed Hassani and George J. Pappas and Eric Wong},
  year = {2025},
  journal = {IEEE SaTML 2025},
  eprint = {2310.08419},
  archivePrefix = {arXiv},
  doi = {10.1109/SaTML64287.2025.00010},
  url = {https://arxiv.org/abs/2310.08419}
}

@misc{antebi2024gpt,
  title = {{GPT in Sheep's Clothing: The Risk of Customized GPTs}},
  author = {Sagiv Antebi and Noam Azulay and Edan Habler and Ben Ganon and Asaf Shabtai and Yuval Elovici},
  year = {2024},
  eprint = {2401.09075},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2401.09075}
}

@misc{deadbits2024vigil,
  title = {{Vigil: LLM Prompt Injection Detection and Defense Toolkit}},
  author = {DeadBits},
  year = {2024},
  url = {https://github.com/deadbits/vigil-llm}
}

@misc{guardrails2024guardrails,
  title = {{Guardrails AI: Input/Output Guards for LLM Applications}},
  author = {{Guardrails AI}},
  year = {2024},
  url = {https://github.com/guardrails-ai/guardrails}
}

@misc{protect2024llm,
  title = {{LLM Guard: Security Toolkit for LLM Interactions}},
  author = {{Protect AI}},
  year = {2024},
  url = {https://github.com/protectai/llm-guard}
}

@misc{he2024emerged,
  title = {{The Emerged Security and Privacy of LLM Agent: A Survey with Case Studies}},
  author = {Feng He and Tianqing Zhu and Dayong Ye and Bo Liu and Wanlei Zhou and Philip S. Yu},
  year = {2024},
  eprint = {2407.19354},
  archivePrefix = {arXiv},
  doi = {10.1145/3773080},
  url = {https://arxiv.org/abs/2407.19354}
}

@inproceedings{andriushchenko2025jailbreaking,
  title = {{Jailbreaking Leading Safety-Aligned LLMs with Simple Adaptive Attacks}},
  author = {Maksym Andriushchenko and Francesco Croce and Nicolas Flammarion},
  year = {2025},
  booktitle = {ICLR 2025},
  eprint = {2404.02151},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2404.02151}
}

@inproceedings{zeng2024how,
  title = {{How Johnny Can Persuade LLMs to Jailbreak Them: Rethinking Persuasion to Challenge AI Safety}},
  author = {Yi Zeng and Hongpeng Lin and Jingwen Zhang and Diyi Yang and Ruoxi Jia and Weiyan Shi},
  year = {2024},
  booktitle = {ACL 2024},
  eprint = {2401.06373},
  archivePrefix = {arXiv},
  doi = {10.18653/v1/2024.acl-long.773},
  url = {https://arxiv.org/abs/2401.06373}
}

@techreport{anthropic2024modelb,
  title = {{Model Context Protocol (MCP): Specification}},
  author = {{Anthropic}},
  year = {2024},
  institution = {Anthropic / GitHub},
  url = {https://modelcontextprotocol.io/}
}

@techreport{anthropic2024anthropics,
  title = {{Anthropic's Responsible Scaling Policy}},
  author = {{Anthropic}},
  year = {2024},
  institution = {Anthropic Blog},
  url = {https://www.anthropic.com/news/anthropics-responsible-scaling-policy}
}

@techreport{openai2023openai,
  title = {{OpenAI: Preparedness Framework (Beta)}},
  author = {{OpenAI}},
  year = {2023},
  institution = {OpenAI Blog},
  url = {https://openai.com/safety/preparedness}
}

@techreport{parliament2024eu,
  title = {{EU AI Act: Regulation on Artificial Intelligence}},
  author = {European Parliament},
  year = {2024},
  institution = {Official Journal of the European Union},
  url = {https://eur-lex.europa.eu/eli/reg/2024/1689/oj}
}

@techreport{google2023google,
  title = {{Google: Secure AI Framework (SAIF)}},
  author = {{Google}},
  year = {2023},
  institution = {Google Security Blog},
  url = {https://safety.google/cybersecurity-advancements/saif/}
}

@techreport{bullwinkel2025lessons,
  title = {{Lessons From Red Teaming 100 Generative AI Products}},
  author = {Blake Bullwinkel and Amanda Minnich and Shiven Chawla and Gary Lopez and Martin Pouliot and Whitney Maxwell and Joris de Gruyter and Katherine Pratt and Saphir Qi and Nina Chikanov and Roman Lutz and Raja Sekhar Rao Dheekonda and Bolor-Erdene Jagdagdorj and Eugenia Kim and Justin Song and Keegan Hines and Daniel Jones and Giorgio Severi and Richard Lundeen and Sam Vaughan and Victoria Westerhoff and Pete Bryan and Ram Shankar Siva Kumar and Yonatan Zunger and Chang Kawaguchi and Mark Russinovich},
  year = {2025},
  institution = {arXiv preprint},
  eprint = {2501.07238},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2501.07238}
}

@misc{fang2024llmb,
  title = {{LLM Agents Can Autonomously Exploit One-day Vulnerabilities}},
  author = {Richard Fang and Rohan Bindu and Akul Gupta and Daniel Kang},
  year = {2024},
  eprint = {2404.08144},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2404.08144}
}

@inproceedings{wu2025dissecting,
  title = {{Dissecting Adversarial Robustness of Multimodal LM Agents}},
  author = {Chen Henry Wu and Rishi Shah and Jing Yu Koh and Ruslan Salakhutdinov and Daniel Fried and Aditi Raghunathan},
  year = {2025},
  booktitle = {ICLR 2025},
  eprint = {2406.12814},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2406.12814}
}

@inproceedings{yang2024sweagent,
  title = {{SWE-agent: Agent-Computer Interfaces Enable Automated Software Engineering}},
  author = {John Yang and Carlos E. Jimenez and Alexander Wettig and Kilian Lieret and Shunyu Yao and Karthik Narasimhan and Ofir Press},
  year = {2024},
  booktitle = {NeurIPS 2024},
  eprint = {2405.15793},
  archivePrefix = {arXiv},
  doi = {10.52202/079017-1601},
  url = {https://arxiv.org/abs/2405.15793}
}

@inproceedings{sun2024trustllm,
  title = {{TrustLLM: Trustworthiness in Large Language Models}},
  author = {Lichao Sun and Yue Huang and Haoran Wang and Siyuan Wu and Qihui Zhang},
  year = {2024},
  booktitle = {ICML 2024},
  eprint = {2401.05561},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2401.05561}
}

@misc{kapoor2024societal,
  title = {{On the Societal Impact of Open Foundation Models}},
  author = {Sayash Kapoor and Rishi Bommasani and Kevin Klyman and Shayne Longpre and Ashwin Ramaswami},
  year = {2024},
  eprint = {2403.07918},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2403.07918}
}

@inproceedings{carlini2024poisoning,
  title = {{Poisoning Web-Scale Training Datasets is Practical}},
  author = {Nicholas Carlini and Matthew Jagielski and Christopher A. Choquette-Choo and Daniel Paleka and Will Pearce and Hyrum Anderson and Andreas Terzis and Kurt Thomas and Florian Tramer},
  year = {2024},
  booktitle = {IEEE S\&P 2024},
  eprint = {2302.10149},
  archivePrefix = {arXiv},
  doi = {10.1109/sp54263.2024.00179},
  url = {https://arxiv.org/abs/2302.10149}
}

@inproceedings{qi2024visual,
  title = {{Visual Adversarial Examples Jailbreak Aligned Large Language Models}},
  author = {Xiangyu Qi and Kaixuan Huang and Ashwinee Panda and Peter Henderson and Mengdi Wang and Prateek Mittal},
  year = {2024},
  booktitle = {AAAI 2024},
  eprint = {2306.13213},
  archivePrefix = {arXiv},
  doi = {10.1609/aaai.v38i19.30150},
  url = {https://arxiv.org/abs/2306.13213}
}

@inproceedings{carlini2024are,
  title = {{Are Aligned Neural Networks Adversarially Aligned?}},
  author = {Nicholas Carlini and Milad Nasr and Christopher A. Choquette-Choo and Matthew Jagielski and Irena Gao and Anas Awadalla and Pang Wei Koh and Daphne Ippolito and Katherine Lee and Florian Tramer and Ludwig Schmidt},
  year = {2024},
  booktitle = {NeurIPS 2023},
  eprint = {2306.15447},
  archivePrefix = {arXiv},
  doi = {10.52202/075280-2687},
  url = {https://arxiv.org/abs/2306.15447}
}

@book{huang2024generative,
  title = {{Generative AI Security: Theories and Practices}},
  author = {Ken Huang and Yang Wang and Ben Goertzel and Yale Li and Sean Wright and Jyoti Ponnapalli},
  year = {2024},
  publisher = {Springer},
  url = {https://link.springer.com/book/10.1007/978-3-031-54252-7}
}

@inproceedings{yao2023react,
  title = {{ReAct: Synergizing Reasoning and Acting in Language Models}},
  author = {Shunyu Yao and Jeffrey Zhao and Dian Yu and Nan Du and Izhak Shafran and Karthik Narasimhan and Yuan Cao},
  year = {2023},
  booktitle = {ICLR 2023},
  eprint = {2210.03629},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2210.03629}
}

@inproceedings{wang2023voyager,
  title = {{Voyager: An Open-Ended Embodied Agent with Large Language Models}},
  author = {Guanzhi Wang and Yuqi Xie and Yunfan Jiang and Ajay Mandlekar and Chaowei Xiao and Yuke Zhu and Linxi Fan and Anima Anandkumar},
  year = {2023},
  booktitle = {NeurIPS 2023},
  eprint = {2305.16291},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2305.16291}
}

@misc{zhang2024safetybench,
  title = {{SafetyBench: Evaluating the Safety of Large Language Models}},
  author = {Zhexin Zhang and Leqi Lei and Lindong Wu and Rui Sun and Yongkang Huang},
  year = {2024},
  eprint = {2309.07045},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2309.07045}
}

@inproceedings{patil2024can,
  title = {{Can Sensitive Information Be Deleted From LLMs? Objectives for Defending Against Extraction Attacks}},
  author = {Vaidehi Patil and Peter Hase and Mohit Bansal},
  year = {2024},
  booktitle = {ICLR 2024},
  eprint = {2309.17410},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2309.17410}
}

@misc{han2024wildguard,
  title = {{WildGuard: Open One-Stop Moderation Tools for Safety Risks, Jailbreaks, and Refusals of LLMs}},
  author = {Seungju Han and Kavel Rao and Allyson Ettinger and Liwei Jiang and Bill Yuchen Lin and Nathan Lambert and Yejin Choi and Nouha Dziri},
  year = {2024},
  eprint = {2406.18495},
  archivePrefix = {arXiv},
  doi = {10.52202/079017-0261},
  url = {https://arxiv.org/abs/2406.18495}
}

@inproceedings{souly2024strongreject,
  title = {{A StrongREJECT for Empty Jailbreaks}},
  author = {Alexandra Souly and Qingyuan Lu and Dillon Bowen and Tu Trinh and Elvis Hsieh and Sana Pandey and Pieter Abbeel and Justin Svegliato and Scott Emmons and Olivia Watkins and Sam Toyer},
  year = {2024},
  booktitle = {NeurIPS 2024 Datasets and Benchmarks},
  eprint = {2402.10260},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2402.10260}
}

@misc{owasp2024owaspb,
  title = {{OWASP Threat Dragon: AI-Aware Threat Modeling Tool}},
  author = {{OWASP Foundation}},
  year = {2024},
  url = {https://owasp.org/www-project-threat-dragon/}
}

@techreport{cybersecurity2023cisa,
  title = {{CISA: Roadmap for Artificial Intelligence}},
  author = {{Cybersecurity and Infrastructure Security Agency}},
  year = {2023},
  month = nov,
  institution = {CISA},
  url = {https://www.cisa.gov/resources-tools/resources/roadmap-ai}
}

@inproceedings{shen2024prompt,
  title = {{Prompt Stealing Attacks Against Text-to-Image Generation Models}},
  author = {Xinyue Shen and Yiting Qu and Michael Backes and Yang Zhang},
  year = {2024},
  booktitle = {USENIX Security 2024},
  eprint = {2302.09923},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2302.09923}
}

@misc{yang2023shadow,
  title = {{Shadow Alignment: The Ease of Subverting Safely-Aligned Language Models}},
  author = {Xianjun Yang and Xiao Wang and Qi Zhang and Linda Petzold and William Yang Wang and Xun Zhao and Dahua Lin},
  year = {2023},
  eprint = {2310.02949},
  archivePrefix = {arXiv},
  url = {https://arxiv.org/abs/2310.02949}
}