Skip to content
Merged
  •  
  •  
  •  
11 changes: 7 additions & 4 deletions doc/references.bib
Original file line number Diff line number Diff line change
Expand Up @@ -484,12 +484,15 @@ @article{ji2023beavertails
url = {https://arxiv.org/abs/2307.04657},
}

@article{shen2023donotanything,
@inproceedings{shen2023donotanything,
title = {``Do Anything Now'': Characterizing and Evaluating In-The-Wild Jailbreak Prompts on Large Language Models},
author = {Xinyue Shen and Zeyuan Chen and Michael Backes and Yun Shen and Yang Zhang},
journal = {arXiv preprint arXiv:2308.03825},
year = {2023},
url = {https://arxiv.org/abs/2308.03825},
booktitle = {Proceedings of the 2024 ACM SIGSAC Conference on Computer and Communications Security},
pages = {1671--1685},
year = {2024},
url = {https://doi.org/10.1145/3658644.3670388},
doi = {10.1145/3658644.3670388},
note = {Official dataset: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts},
}

@article{aakanksha2024multilingual,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_00118a609b9a9e63
description: 'Community-posted jailbreak prompt collected in the wild from the discord source ''ChatGPT'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the embedded sample request at its natural final user-request position was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_00f76622c8593aff
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''jailbreak_chat'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the original user-request placeholder was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_010edc598ca578db
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTPromptGenius'' (paper community: Basic) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_025f8aaf6d855285
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_0268a88a13d5d082
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''jailbreak_chat'' (paper community: Start Prompt) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the original user-request placeholder was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_03278e7112d79d1b
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPT'' (paper community: Basic) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_033586ced7abc818
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the embedded sample request at its natural final user-request position was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_03d9ca68e3e61f7b
description: 'Community-posted jailbreak prompt collected in the wild from the discord source ''Spreadsheet Warriors'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_04a21012c7852cb0
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''flowgpt'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_05b154c5cf4b162b
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_05c7b1cb279e16e3
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the original user-request placeholder was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_05d97f7a9bff6163
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTPromptGenius'' (paper community: Basic) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the embedded sample request at its natural final user-request position was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_064b882b4d6e9903
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the original user-request placeholder was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_0659b10391be3b4c
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''jailbreak_chat'' (paper community: Advanced) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the original user-request placeholder was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_067388c265900944
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_06b6d3c518c70a6d
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''flowgpt'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_077d43d9110a0502
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''jailbreak_chat'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the original user-request placeholder was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_07ee0d851a4c153a
description: 'Community-posted jailbreak prompt collected in the wild from the discord source ''ChatGPT'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_07fb723484b6b667
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''jailbreak_chat'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the original user-request placeholder was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_0844dee001544bd3
description: 'Community-posted jailbreak prompt collected in the wild from the discord source ''ChatGPT'' (paper community: Anarchy) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_084ea70adfbf8331
description: 'Community-posted jailbreak prompt collected in the wild from the discord source ''ChatGPT'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: the embedded sample request at its natural final user-request position was replaced with a Jinja {{ prompt }} slot. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_086aeef18dd9e65a
description: 'Community-posted jailbreak prompt collected in the wild from the website source ''flowgpt'' (paper community: Guidelines) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_087b57ff8352ef62
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' (paper community: Advanced) by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_099b91b079b07fa9
description: 'Community-posted jailbreak prompt collected in the wild from the reddit source ''ChatGPTJailbreak'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@ name: trustairlab_0a11a04ee05afbfe
description: 'Community-posted jailbreak prompt collected in the wild from the discord source ''AI Prompt Sharing'' by the TrustAIRLab ''In-The-Wild Jailbreak Prompts'' study (Shen et al., 2024) and curated by that dataset''s authors. It has been converted into a reusable single-slot jailbreak wrapper: a Jinja {{ prompt }} slot was appended at its natural final user-request position so a fresh user request can be inserted. Original user identities and handles are not included.'
authors: []
groups:
- TrustAIRLab
- CISPA Helmholtz Center for Information Security
- NetApp
source: https://huggingface.co/datasets/TrustAIRLab/in-the-wild-jailbreak-prompts
dataset_name: trustairlab_in_the_wild_jailbreak_2023_12_25
parameters:
Expand Down
Loading