{ "catalog": "RAG Repo source directory", "site": "https://rag-repo.org", "generated": "2026-09-16", "count": 238, "licence": "Catalogue metadata is provided for reuse with attribution to RAG Repo (rag-repo.org). Each source carries its own licence in the \"licence\" field.", "legend": { "accessType": { "open": "freely accessible, open licence", "limited": "free but conditional (registration, credentialing, or a free tier)", "commercial": "primarily paid, or a marketplace" }, "accessPatterns": { "_note": "how you get the bytes; a source may have several", "values": [ "huggingface", "cloud-bucket", "bulk-download", "api", "git-repo", "web-portal", "web-crawl" ], "huggingface": "via the Hugging Face Hub / datasets library", "cloud-bucket": "public S3 / GCS / Azure bucket, bulk objects", "bulk-download": "direct file downloads or dumps over HTTP", "api": "REST / GraphQL / SPARQL query API (fetch slices)", "git-repo": "cloned from GitHub / Software Heritage", "web-portal": "a portal or marketplace, browse then download per-dataset", "web-crawl": "no structured export; must be scraped from the site" }, "sizeTier": { "_note": "whole-corpus download volume if fully pulled", "values": [ "tiny", "small", "medium", "large", "huge", "unknown" ], "tiny": "under 1 GB", "small": "1 GB to 50 GB", "medium": "50 GB to 1 TB", "large": "1 TB to 50 TB", "huge": "over 50 TB", "unknown": "not stated, or a live query API with no single size" }, "ragReadiness": { "raw": "as collected, needs cleaning", "cleaned": "filtered and tidied, not yet split for retrieval", "chunked": "already split into passages", "embedded": "ships with precomputed vectors", "mixed": "offered in several of these states" }, "version": "the edition of the source we describe (null if unversioned or rolling)", "lastReviewed": "YYYY-MM-DD when we last confirmed this entry", "supersedes": "slugs of older editions this one replaces", "supersededBy": "slug of the newer edition that replaces this one (null if current)" }, "sources": [ { "slug": "ai4bharat", "name": "AI4Bharat (IndicCorp)", "page": "https://rag-repo.org/source/ai4bharat", "url": "https://ai4bharat.iitm.ac.in", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "medium", "licence": "Varies by dataset (mostly permissive)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "IIT Madras", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "indic-languages", "india", "parallel-corpus", "benchmarks" ], "description": "A collection of corpora, models, and benchmarks for Indian languages, produced by IIT Madras. It covers major Indic languages with monolingual corpora, parallel translation data, and evaluation sets.", "links": {} }, { "slug": "alphafold-protein-structure-database", "name": "AlphaFold Protein Structure Database", "page": "https://rag-repo.org/source/alphafold-protein-structure-database", "url": "https://alphafold.ebi.ac.uk", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "open", "accessPatterns": [ "api", "bulk-download", "cloud-bucket" ], "ragReadiness": "raw", "format": [ "PDB", "mmCIF", "JSON" ], "size": "200M+ predicted structures", "sizeTier": "large", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "EMBL-EBI and Google DeepMind", "updateFrequency": "Periodic", "yearLaunched": 2021, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "proteins", "structure-prediction", "bioinformatics", "deepmind", "api", "bulk-download", "medical-research" ], "description": "EMBL-EBI and Google DeepMind's open database of AI-predicted protein structures, covering over 200 million sequences across almost all of UniProt. Each model ships with per-residue confidence and predicted aligned error scores, and is reachable by website, API, FTP and Google Cloud bulk access. Released under CC BY 4.0 and designed to pair with UniProt and the experimental structures in the PDB.", "links": { "api": "https://alphafold.ebi.ac.uk/api-docs" } }, { "slug": "amps", "name": "AMPS", "page": "https://rag-repo.org/source/amps", "url": "https://github.com/hendrycks/math", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "git-repo", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "LaTeX", "JSON" ], "size": "~23 GB", "sizeTier": "small", "licence": "MIT", "licenceUrl": "https://github.com/hendrycks/math/blob/main/LICENSE", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Dan Hendrycks", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "problem-solving", "latex", "khan-academy", "step-by-step" ], "description": "A dataset of informal mathematics introduced alongside the MATH benchmark. It includes more than 100,000 Khan Academy problems with step-by-step solutions in LaTeX and over 5 million problems generated with Mathematica scripts, totalling around 23 GB.", "links": {} }, { "slug": "arxiv", "name": "arXiv", "page": "https://rag-repo.org/source/arxiv", "url": "https://arxiv.org/help/bulk_data", "category": "academic", "categoryName": "Academic & scientific literature", "accessType": "open", "accessPatterns": [ "cloud-bucket", "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "PDF", "TeX", "XML" ], "size": "2.5M+ papers", "sizeTier": "large", "licence": "Varies by paper (mostly arXiv non-exclusive licence)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Cornell University", "updateFrequency": "Daily", "yearLaunched": 1991, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "preprints", "physics", "mathematics", "computer-science", "research", "bulk-download" ], "description": "An open-access preprint server for physics, mathematics, computer science, quantitative biology, statistics, and more, holding over 2.5 million papers. You can pull the full archive in bulk from Amazon S3, or harvest the metadata through OAI-PMH, a standard protocol for sharing records between repositories.", "links": { "documentation": "https://arxiv.org/help/bulk_data" } }, { "slug": "austlii", "name": "AustLII", "page": "https://rag-repo.org/source/austlii", "url": "https://www.austlii.edu.au", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "web-crawl" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "AustLII terms (free access)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "case-law", "australia", "legislation", "court-opinions", "legal" ], "description": "A free resource of medium-neutral case law and unreported judgments for all Australian jurisdictions, covering the whole country since 1995.", "links": {} }, { "slug": "automathtext", "name": "AutoMathText", "page": "https://rag-repo.org/source/automathtext", "url": "https://huggingface.co/datasets/math-ai/AutoMathText", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~200 GB", "sizeTier": "medium", "licence": "CC BY-SA 4.0", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "quality-scored", "arxiv", "github", "pretraining" ], "description": "Around 200 GB of mathematical text compiled from websites, arXiv, and GitHub, drawing on OpenWebMath, RedPajama, and AlgebraicStack. Every piece of content carries a score from 0 to 1 for relevance, quality, and educational value, labelled automatically by the Qwen-72B model.", "links": {} }, { "slug": "awesome-ai-math-datasets", "name": "Awesome AI Math Datasets", "page": "https://rag-repo.org/source/awesome-ai-math-datasets", "url": "https://github.com/amao0o0/awesome-AI-Math-Datasets", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "tiny", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "curated-list", "meta-resource", "index" ], "description": "A community-curated list of open-source mathematics datasets for training and evaluating maths-capable language models. A useful index for finding newer additions in this space.", "links": {} }, { "slug": "awesome-legal-data", "name": "Awesome Legal Data", "page": "https://rag-repo.org/source/awesome-legal-data", "url": "https://github.com/openlegaldata/awesome-legal-data", "category": "curated-lists", "categoryName": "Curated lists & meta-resources", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "raw", "format": [ "Markdown" ], "size": null, "sizeTier": "tiny", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Open Legal Data (community)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "awesome-list", "legal", "directory", "community", "github" ], "description": "A community-maintained list of legal datasets, tools, and resources for legal text processing across jurisdictions, including court records, statutes, contracts, and legal NLP benchmarks. A useful map for anyone building a legal RAG system.", "links": { "github": "https://github.com/openlegaldata/awesome-legal-data" } }, { "slug": "awesome-legaltech", "name": "Awesome LegalTech", "page": "https://rag-repo.org/source/awesome-legaltech", "url": "https://github.com/Vaquill-AI/awesome-legaltech", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "tiny", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Vaquill AI", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "meta-resource", "curated-list", "legaltech", "legal", "datasets" ], "description": "A curated list of legal technology resources: open-source platforms, AI models, companies, datasets, and tools spanning the global legal ecosystem. Useful for tracking new entrants in a fast-moving space.", "links": { "github": "https://github.com/Vaquill-AI/awesome-legaltech" } }, { "slug": "awesome-public-datasets", "name": "Awesome Public Datasets", "page": "https://rag-repo.org/source/awesome-public-datasets", "url": "https://github.com/awesomedata/awesome-public-datasets", "category": "curated-lists", "categoryName": "Curated lists & meta-resources", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "raw", "format": [ "Markdown" ], "size": null, "sizeTier": "tiny", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "awesomedata (community)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "awesome-list", "directory", "open-data", "community", "github" ], "description": "A community-curated list of high-quality open datasets on GitHub, organised by topic: agriculture, biology, climate, economics, education, finance, government, healthcare, and more. A good starting point when you need RAG-ready data for a specific domain and do not yet know where to look.", "links": { "github": "https://github.com/awesomedata/awesome-public-datasets" } }, { "slug": "aws-data-exchange", "name": "AWS Data Exchange", "page": "https://rag-repo.org/source/aws-data-exchange", "url": "https://aws.amazon.com/data-exchange/", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "commercial", "accessPatterns": [ "web-portal", "cloud-bucket", "api" ], "ragReadiness": "raw", "format": [ "CSV", "JSON", "Parquet" ], "size": null, "sizeTier": "unknown", "licence": "Varies by listing", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Amazon Web Services", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "data-platform", "marketplace", "commercial", "cloud", "third-party-data" ], "description": "A marketplace for finding, subscribing to, and using third-party data inside the AWS cloud. It carries both free open datasets and paid commercial data products, so you can pull licensed data straight into your AWS workflows without setting up separate transfers.", "links": {} }, { "slug": "aws-open-data", "name": "AWS Open Data Registry", "page": "https://rag-repo.org/source/aws-open-data", "url": "https://registry.opendata.aws", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "open", "accessPatterns": [ "web-portal", "cloud-bucket" ], "ragReadiness": "raw", "format": [ "Parquet", "CSV", "JSON" ], "size": "300+ PB", "sizeTier": "huge", "licence": "Varies by dataset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Amazon Web Services", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "data-platform", "cloud", "geospatial", "genomics", "satellite", "climate" ], "description": "A registry of high-value datasets hosted on AWS and made publicly available, covering genomics, geospatial data, climate, satellite imagery, and more. Over 300 PB of data in total, free to access: you pay only for the compute you use to process it.", "links": {} }, { "slug": "bailii", "name": "BAILII (British and Irish Legal Information Institute)", "page": "https://rag-repo.org/source/bailii", "url": "https://www.bailii.org", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "web-crawl" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Crown copyright / Open Government Licence (most content)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "British and Irish Legal Information Institute", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "case-law", "uk", "ireland", "legislation", "legal" ], "description": "Free access to British and Irish primary legal materials, covering UK and Ireland case law and legislation. Alongside the National Archives, one of the main free sources for reading UK judgments.", "links": {} }, { "slug": "beir", "name": "BEIR", "page": "https://rag-repo.org/source/beir", "url": "https://github.com/beir-cellar/beir", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "git-repo", "bulk-download" ], "ragReadiness": "chunked", "format": [], "size": "18 datasets", "sizeTier": "small", "licence": "Apache 2.0 (framework); individual datasets vary", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "information-retrieval", "zero-shot", "evaluation" ], "description": "A collection of 18 information retrieval datasets spanning ad-hoc web search, question answering, fact verification, and duplicate question retrieval. Built by aggregating existing datasets, some originally created for other tasks and converted to retrieval format. Tests how well retrieval models generalise to unseen domains without fine-tuning, which matters for real-world RAG.", "links": {} }, { "slug": "bfcl", "name": "Berkeley Function-Calling Leaderboard (BFCL)", "page": "https://rag-repo.org/source/bfcl", "url": "https://gorilla.cs.berkeley.edu/leaderboard.html", "category": "agentic-tool-use", "categoryName": "Agentic & Tool-Use", "accessType": "open", "accessPatterns": [ "git-repo", "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": "~12 MB of JSON cases", "sizeTier": "tiny", "licence": "Apache 2.0", "licenceUrl": "https://www.apache.org/licenses/LICENSE-2.0", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Gorilla project, UC Berkeley", "updateFrequency": "Periodic", "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "function-calling", "tool-use", "agentic", "benchmark", "evaluation", "multi-turn" ], "description": "The de facto benchmark for how well language models call functions, APIs and tools. Built by UC Berkeley's Gorilla project, it spans Python, Java, JavaScript and REST with simple, parallel, irrelevance-detection, multi-turn and agentic cases. Apache 2.0 and freely available.", "links": { "documentation": "https://github.com/ShishirPatil/gorilla/blob/main/berkeley-function-call-leaderboard/README.md", "github": "https://github.com/ShishirPatil/gorilla", "huggingface": "https://huggingface.co/datasets/gorilla-llm/Berkeley-Function-Calling-Leaderboard" } }, { "slug": "bigpatent", "name": "BIGPATENT", "page": "https://rag-repo.org/source/bigpatent", "url": "https://huggingface.co/datasets/NortheasternUniversity/big_patent", "category": "patents", "categoryName": "Patents & intellectual property", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "1.3M patents (~34 GB)", "sizeTier": "small", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Northeastern University", "updateFrequency": "Archived", "yearLaunched": 2019, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "patents", "us", "uspto", "summarization", "nlp", "machine-learning" ], "description": "A corpus of 1.3 million US utility patents filed between 1971 and 2018, each paired with its human-written abstract as a gold-standard summary and organised by Cooperative Patent Classification code. A large, clean patent text corpus built for abstractive summarisation and other patent NLP work.", "links": { "huggingface": "https://huggingface.co/datasets/NortheasternUniversity/big_patent" } }, { "slug": "bio2rdf", "name": "Bio2RDF", "page": "https://rag-repo.org/source/bio2rdf", "url": "https://bio2rdf.org", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "RDF" ], "size": "11 billion triples across 35 datasets", "sizeTier": "unknown", "licence": "Varies by source dataset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "life-sciences", "knowledge-graph", "sparql", "rdf", "linked-data" ], "description": "An open-source project that pulls together a diverse set of life-sciences datasets from many providers into a single linked-data graph, with a SPARQL endpoint for querying across them. The full collection is about 11 billion triples across 35 datasets, including DrugBank, PubMed, and MeSH.", "links": {} }, { "slug": "bright", "name": "BRIGHT", "page": "https://rag-repo.org/source/bright", "url": "https://brightbenchmark.github.io", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "git-repo", "huggingface" ], "ragReadiness": "chunked", "format": [], "size": null, "sizeTier": "tiny", "licence": "CC BY 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "reasoning", "evaluation" ], "description": "A reasoning-intensive retrieval benchmark. It requires genuine reasoning to connect queries with relevant documents rather than keyword or semantic overlap, revealing model weaknesses that BEIR misses.", "links": {} }, { "slug": "c4", "name": "C4 (Colossal Clean Crawled Corpus)", "page": "https://rag-repo.org/source/c4", "url": "https://huggingface.co/datasets/allenai/c4", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet", "JSON" ], "size": "750 GB", "sizeTier": "medium", "licence": "ODC-By 1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Allen Institute for AI (Allen AI)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "english", "pretraining", "cleaned", "allen-ai" ], "description": "A 750 GB English corpus derived from Common Crawl using heuristic cleaning to keep natural language and drop gibberish, boilerplate, and placeholder text. Built to train Google's T5 and later used for MPT-7B and others.", "links": { "huggingface": "https://huggingface.co/datasets/allenai/c4" } }, { "slug": "cambridge-law-corpus", "name": "Cambridge Law Corpus", "page": "https://rag-repo.org/source/cambridge-law-corpus", "url": "https://arxiv.org/abs/2309.12269", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "limited", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "small", "licence": "Research use (restricted access)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "University of Cambridge", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "case-law", "legal", "academic", "restricted-access" ], "description": "A research dataset of more than 250,000 UK court cases, mostly from the 21st century but with some reaching back to the 16th century. Built by the University of Cambridge for legal natural language processing work, it is available under restricted access for research use.", "links": {} }, { "slug": "canlii", "name": "CanLII", "page": "https://rag-repo.org/source/canlii", "url": "https://www.canlii.org", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "limited", "accessPatterns": [ "api", "web-crawl" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "CanLII terms (free access, restrictions on bulk reuse)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "case-law", "canada", "legislation", "bilingual", "legal" ], "description": "A national legal information institute covering Canada's federal and provincial jurisdictions, hosting over 300 databases of legislation and case law. Bilingual, with thousands of commentaries on Canadian court decisions.", "links": {} }, { "slug": "case-network", "name": "CASE Network (1EdTech)", "page": "https://rag-repo.org/source/case-network", "url": "https://casenetwork.1edtech.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": "Frameworks from all 50 US states plus other issuing agencies", "sizeTier": "tiny", "licence": "Free access via 1EdTech; underlying standards carry their issuing agency's terms", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "1EdTech Consortium", "updateFrequency": "Ongoing", "yearLaunched": 2019, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "curriculum", "standards", "us", "k12", "machine-readable", "api", "structured" ], "description": "A public registry of machine-readable learning-standard frameworks from all 50 US states and other issuing agencies, run by 1EdTech in the CASE JSON format. The digitally referenceable spine of what to teach, at which level and in what order, rather than the teaching content itself.", "links": { "documentation": "https://www.1edtech.org/standards/case/about" } }, { "slug": "caselaw-access-project", "name": "Caselaw Access Project", "page": "https://rag-repo.org/source/caselaw-access-project", "url": "https://case.law", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "api", "bulk-download", "huggingface" ], "ragReadiness": "cleaned", "format": [], "size": "6.9M decisions", "sizeTier": "medium", "licence": "Public domain (US court opinions); CC0 for the digitisation", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Harvard Law School Library Innovation Lab", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "case-law", "us", "public-domain", "legal", "court-opinions" ], "description": "6.9M US court decisions spanning the 1600s to 2020, digitised by Harvard Law School Library and released in a consistent machine-readable format. Covers every official, book-published state and federal case, with a free bulk API.", "links": { "huggingface": "https://huggingface.co/datasets/free-law/Caselaw_Access_Project" } }, { "slug": "cc-100", "name": "CC-100", "page": "https://rag-repo.org/source/cc-100", "url": "https://data.statmt.org/cc-100/", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "chunked", "format": [ "TXT" ], "size": null, "sizeTier": "large", "licence": "Common Crawl terms", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "common-crawl", "paragraph-level", "pretraining" ], "description": "A high-quality multilingual corpus derived from Common Crawl. It splits every document into separate paragraphs, so it is effectively a paragraph-level rather than document-level corpus, which matters for your chunking strategy.", "links": {} }, { "slug": "cc-news", "name": "CC-News", "page": "https://rag-repo.org/source/cc-news", "url": "https://commoncrawl.org/blog/news-dataset-available", "category": "news-media", "categoryName": "News, events & media", "accessType": "open", "accessPatterns": [ "cloud-bucket", "huggingface" ], "ragReadiness": "raw", "format": [ "WARC" ], "size": null, "sizeTier": "large", "licence": "As per Common Crawl (content subject to original site terms)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Common Crawl Foundation", "updateFrequency": "Continuous", "yearLaunched": 2016, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "news", "web-crawl", "pretraining", "english", "multilingual", "common-crawl" ], "description": "A subset of Common Crawl focused on news, containing millions of articles pulled from news websites around the world. It has fed several large language model training pipelines and gives you a ready news corpus without crawling sites yourself.", "links": { "huggingface": "https://huggingface.co/datasets/cc_news" } }, { "slug": "chebi", "name": "ChEBI", "page": "https://rag-repo.org/source/chebi", "url": "https://www.ebi.ac.uk/chebi/", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "SDF", "OWL" ], "size": null, "sizeTier": "tiny", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "EMBL-EBI (European Bioinformatics Institute)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "chemistry", "ontology", "controlled-vocabulary", "entity-linking", "structured" ], "description": "A dictionary of small chemical compound molecular entities, with structure files and ontology files available for download. Particularly useful as a controlled vocabulary for chemical entity linking, where you match a mention in text to a standard identifier.", "links": {} }, { "slug": "chembl", "name": "ChEMBL", "page": "https://rag-repo.org/source/chembl", "url": "https://www.ebi.ac.uk/chembl/", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [], "size": "2.5M compound records, 2M unique structures", "sizeTier": "small", "licence": "CC BY-SA 3.0", "licenceUrl": "https://creativecommons.org/licenses/by-sa/3.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "EMBL-EBI (European Bioinformatics Institute)", "updateFrequency": "Every three to four months", "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "chemistry", "drug-discovery", "bioactivity", "curated", "share-alike", "genomics" ], "description": "A manually curated database of bioactive molecules with drug-like properties, bringing together chemical, bioactivity, and genomic data to support drug discovery. Holds close to 2.5M compound records on nearly 2M unique chemical structures, extracted mainly from the medicinal chemistry literature.", "links": {} }, { "slug": "ck-12", "name": "CK-12 Foundation", "page": "https://rag-repo.org/source/ck-12", "url": "https://www.ck12.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "web-crawl" ], "ragReadiness": "cleaned", "format": [ "HTML", "PDF" ], "size": null, "sizeTier": "small", "licence": "CK-12 Curriculum Materials License (based on CC BY-NC)", "licenceUrl": "https://info.ck12.org/terms-of-use", "commercialUse": "prohibited", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "CK-12 Foundation", "updateFrequency": null, "yearLaunched": 2007, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "k-12", "textbooks", "mathematics", "science", "education", "standards-aligned", "non-commercial", "nonprofit" ], "description": "Free K-12 library of customisable FlexBook textbooks, strong in maths and science, with adaptive practice, interactive simulations and study guides, organised by grade level and aligned to state standards. Non-commercial licence.", "links": {} }, { "slug": "clinicaltrials-gov", "name": "ClinicalTrials.gov", "page": "https://rag-repo.org/source/clinicaltrials-gov", "url": "https://clinicaltrials.gov", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "JSON", "XML", "CSV" ], "size": "450,000+ studies", "sizeTier": "small", "licence": "Public domain (US Government)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "US National Institutes of Health", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "clinical-trials", "us", "public-domain", "structured", "api", "medical-research" ], "description": "A US National Institutes of Health database of privately and publicly funded clinical studies. It holds structured records on more than 450,000 studies, including protocols, conditions, interventions, outcomes, and results, all in the public domain.", "links": { "api": "https://clinicaltrials.gov/data-api/api" } }, { "slug": "cohere-wikipedia-22-12", "name": "Cohere Wikipedia 22-12 Embeddings (per-language)", "page": "https://rag-repo.org/source/cohere-wikipedia-22-12", "url": "https://huggingface.co/datasets/Cohere/wikipedia-22-12-en-embeddings", "category": "embedded-datasets", "categoryName": "Pre-embedded & RAG-ready", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "embedded", "format": [], "size": null, "sizeTier": "medium", "licence": "Apache 2.0 (embeddings); CC BY-SA (Wikipedia text)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Cohere", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "2022-12", "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "wikipedia", "embeddings", "multilingual", "semantic-search", "rag-ready", "streaming" ], "description": "Wikipedia encoded with the Cohere multilingual-22-12 embedding model, with embeddings (numeric vectors that capture meaning) computed on each article's title plus text. Published per language, covering many including Arabic, German, English, Spanish, French, Hindi, Italian, Japanese, Korean, Simple English, and Chinese. You can stream it rather than download it in full, which matters given the size.", "links": { "huggingface": "https://huggingface.co/datasets/Cohere/wikipedia-22-12-en-embeddings" } }, { "slug": "cohere-wikipedia-multilingual", "name": "Cohere Wikipedia Multilingual Embeddings (2023-11)", "page": "https://rag-repo.org/source/cohere-wikipedia-multilingual", "url": "https://huggingface.co/datasets/CohereLabs/wikipedia-2023-11-embed-multilingual-v3", "category": "embedded-datasets", "categoryName": "Pre-embedded & RAG-ready", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "embedded", "format": [], "size": "~250 M paragraph embeddings across 300+ languages", "sizeTier": "large", "licence": "Apache 2.0 (embeddings); CC BY-SA (Wikipedia text)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Cohere Labs", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "2023-11", "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "wikipedia", "embeddings", "multilingual", "semantic-search", "rag-ready", "cross-lingual" ], "description": "The November 2023 Wikipedia dump across 300+ languages, split into passages and turned into embeddings (numeric vectors that capture meaning) with the Cohere Embed V3 multilingual model. Close to 250 million paragraph embeddings in total. Drop it into a vector database for semantic search over all of Wikipedia, or use it as a knowledge source for multilingual RAG, including cross-lingual search where a query in one language returns relevant results from another.", "links": { "huggingface": "https://huggingface.co/datasets/CohereLabs/wikipedia-2023-11-embed-multilingual-v3" } }, { "slug": "coir", "name": "CoIR (Code Retrieval Benchmark)", "page": "https://rag-repo.org/source/coir", "url": "https://github.com/CoIR-team/coir", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "git-repo", "huggingface" ], "ragReadiness": "chunked", "format": [], "size": null, "sizeTier": "small", "licence": "MIT", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "code", "evaluation" ], "description": "A code retrieval benchmark spanning diverse programming tasks and languages. Built for evaluating how well models retrieve relevant source code, which matters if you are building RAG over codebases.", "links": {} }, { "slug": "common-core", "name": "Common Core State Standards", "page": "https://rag-repo.org/source/common-core", "url": "https://www.thecorestandards.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "bulk-download", "git-repo" ], "ragReadiness": "cleaned", "format": [ "PDF", "JSON", "XML", "CSV" ], "size": null, "sizeTier": "tiny", "licence": "Copyright NGA Center and CCSSO; public licence permits copying, publishing and display for purposes supporting the initiative, with attribution", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "NGA Center for Best Practices and CCSSO", "updateFrequency": null, "yearLaunched": 2010, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "curriculum", "standards", "k-12", "mathematics", "english-language-arts", "machine-readable", "us" ], "description": "The US Common Core State Standards for English language arts and mathematics, defined grade by grade from kindergarten to grade twelve. A widely adopted curriculum framework, available in machine-readable form through ASN and CASE.", "links": {} }, { "slug": "common-corpus", "name": "Common Corpus", "page": "https://rag-repo.org/source/common-corpus", "url": "https://huggingface.co/datasets/PleIAs/common_corpus", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "2.27 T tokens", "sizeTier": "large", "licence": "Varies (public domain and permissive licences)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "PleIAs", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "pretraining", "public-domain", "permissive-licence", "large-corpus", "provenance" ], "description": "An open, multilingual pretraining corpus of roughly 2.27 trillion tokens, built only from public-domain and permissively licensed text with documented provenance for every document. Maintained by PleIAs.", "links": { "huggingface": "https://huggingface.co/datasets/PleIAs/common_corpus" } }, { "slug": "common-crawl", "name": "Common Crawl", "page": "https://rag-repo.org/source/common-crawl", "url": "https://commoncrawl.org", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "cloud-bucket", "bulk-download" ], "ragReadiness": "raw", "format": [ "WARC", "WET", "WAT" ], "size": "Petabytes (each monthly crawl is 200 to 400 TB compressed)", "sizeTier": "huge", "licence": "Open (content subject to original site terms)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Common Crawl Foundation", "updateFrequency": "Monthly", "yearLaunched": 2007, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "english", "multilingual", "pretraining", "nonprofit" ], "description": "A nonprofit that crawls the web and freely provides its archives and datasets. Petabytes of raw web data from billions of pages, with a new snapshot each month. Used in the training of GPT-3, LLaMA, T5, and many other large language models.", "links": { "documentation": "https://commoncrawl.org/get-started", "download": "https://commoncrawl.org/get-started" } }, { "slug": "cultural-heritage-data-space", "name": "Common European Data Space for Cultural Heritage", "page": "https://rag-repo.org/source/cultural-heritage-data-space", "url": "https://www.dataspace-culturalheritage.eu", "category": "cultural-heritage", "categoryName": "Cultural heritage & archives", "accessType": "open", "accessPatterns": [ "api", "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Varies", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Europeana Initiative", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "cultural-heritage", "europe", "infrastructure", "europeana", "open-data" ], "description": "The umbrella project coordinating cultural heritage data across Europe. Led by the Europeana Initiative, it helps cultural institutions and EU Member States share their collections through a common, shared infrastructure.", "links": {} }, { "slug": "common-pile", "name": "Common Pile v0.1", "page": "https://rag-repo.org/source/common-pile", "url": "https://github.com/r-three/common-pile", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "Parquet", "JSON" ], "size": "8 TB", "sizeTier": "large", "licence": "Varies (open licences and public domain)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "EleutherAI", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": "v0.1", "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "pretraining", "public-domain", "openly-licensed", "large-corpus", "nonprofit", "english" ], "description": "An 8 TB corpus of public domain and openly licensed text drawn from 30 sources, assembled by EleutherAI as a licence-transparent alternative to web-scraped pretraining data. Used to train the Comma family of language models.", "links": { "documentation": "https://blog.eleuther.ai/common-pile/", "github": "https://github.com/r-three/common-pile", "huggingface": "https://huggingface.co/common-pile" } }, { "slug": "companies-house", "name": "Companies House", "page": "https://rag-repo.org/source/companies-house", "url": "https://www.gov.uk/government/organisations/companies-house", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "CSV", "JSON" ], "size": null, "sizeTier": "unknown", "licence": "Open Government Licence v3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Companies House", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "government", "companies", "business", "open-data" ], "description": "The UK's official register of companies. Search every registered company, its directors, filing history, and accounts, or pull the data in bulk. A reliable source for UK corporate facts in a RAG system.", "links": {} }, { "slug": "conceptnet", "name": "ConceptNet", "page": "https://rag-repo.org/source/conceptnet", "url": "https://conceptnet.io", "category": "knowledge-graphs", "categoryName": "Knowledge graphs & structured data", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "mixed", "format": [ "JSON", "CSV" ], "size": null, "sizeTier": "small", "licence": "CC BY-SA 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Luminoso and the Commonsense Computing Initiative", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "knowledge-graph", "common-sense", "multilingual", "words", "relationships" ], "description": "A multilingual common-sense knowledge graph that links words and phrases with labelled connections, such as \"a cat is a pet\" or \"rain causes wet\". It captures the everyday relationships between ideas that plain text rarely spells out.", "links": { "api": "https://api.conceptnet.io", "github": "https://github.com/commonsense/conceptnet5" } }, { "slug": "conceptual-captions", "name": "Conceptual Captions (CC3M / CC12M)", "page": "https://rag-repo.org/source/conceptual-captions", "url": "https://ai.google.com/research/ConceptualCaptions/", "category": "multimodal", "categoryName": "Multimodal & image-text", "accessType": "limited", "accessPatterns": [ "bulk-download", "web-crawl" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "large", "licence": "Google Research terms (permissive for research)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Google Research", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "image-text", "captioning", "alt-text", "google", "multimodal" ], "description": "A cleaned image alt-text dataset for automatic image captioning, produced by Google Research. Alt-text pulled from the web is filtered and generalised (hypernymed), replacing specific names with broader categories. Available in 3.3M (CC3M) and 12M (CC12M) variants.", "links": {} }, { "slug": "copernicus", "name": "Copernicus (EU Earth Observation)", "page": "https://rag-repo.org/source/copernicus", "url": "https://www.copernicus.eu", "category": "climate-earth", "categoryName": "Climate & earth observation", "accessType": "open", "accessPatterns": [ "api", "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Copernicus open licence (free, full, and open)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "European Union / ESA", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "earth-observation", "satellite", "climate", "eu", "sentinel" ], "description": "The EU's flagship Earth observation programme. The Copernicus Climate Change Service portal provides authoritative information about past, present, and future global climate, derived from Sentinel reference products alongside other satellite observations and in-situ measurements.", "links": {} }, { "slug": "core", "name": "CORE", "page": "https://rag-repo.org/source/core", "url": "https://core.ac.uk", "category": "academic", "categoryName": "Academic & scientific literature", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "JSON", "PDF" ], "size": "300M+ records, 40M+ full-text articles", "sizeTier": "large", "licence": "Varies by paper; metadata freely available", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "open-access", "aggregator", "full-text", "research", "api", "bulk-download" ], "description": "An aggregator of open-access research papers that harvests from thousands of repositories and journals worldwide. It holds over 300 million metadata records and more than 40 million full-text articles, all reachable through one search API.", "links": {} }, { "slug": "crag-benchmark", "name": "CRAG (Comprehensive RAG Benchmark)", "page": "https://rag-repo.org/source/crag-benchmark", "url": "https://github.com/facebookresearch/CRAG", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "git-repo", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "JSONL" ], "size": "4,409 QA pairs", "sizeTier": "small", "licence": "CC BY-NC 4.0", "licenceUrl": "https://creativecommons.org/licenses/by-nc/4.0/", "commercialUse": "prohibited", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Meta", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "rag-evaluation", "question-answering", "benchmark", "factual-qa", "knowledge-graph", "web-search" ], "description": "Meta's factual question answering benchmark of 4,409 QA pairs across five domains, paired with mock web-search and knowledge-graph retrieval APIs. It is distinct from \"Corrective RAG (CRAG)\", which is a retrieval method, not a dataset.", "links": { "documentation": "https://github.com/facebookresearch/CRAG/blob/main/docs/dataset.md", "github": "https://github.com/facebookresearch/CRAG" } }, { "slug": "crossref", "name": "Crossref", "page": "https://rag-repo.org/source/crossref", "url": "https://www.crossref.org", "category": "academic", "categoryName": "Academic & scientific literature", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "JSON", "XML" ], "size": "150M+ works", "sizeTier": "medium", "licence": "CC0 (metadata)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Crossref", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "metadata", "doi", "citations", "nonprofit", "research", "api" ], "description": "A nonprofit DOI registration agency that publishes metadata for over 150 million scholarly works, including journal articles, books, and conference proceedings. A DOI is the permanent identifier assigned to each work, and Crossref shares its metadata through a free API.", "links": {} }, { "slug": "crystallography-open-database", "name": "Crystallography Open Database", "page": "https://rag-repo.org/source/crystallography-open-database", "url": "https://www.crystallography.net", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "CIF" ], "size": "350,000+ crystal structure files", "sizeTier": "small", "licence": "CC0 1.0", "licenceUrl": "https://creativecommons.org/publicdomain/zero/1.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "crystallography", "materials-science", "public-domain", "structures" ], "description": "A collection of over 350,000 crystal structure files covering organic, inorganic, and metal-organic compounds, released into the public domain under CC0.", "links": {} }, { "slug": "culturax", "name": "CulturaX", "page": "https://rag-repo.org/source/culturax", "url": "https://huggingface.co/datasets/uonlp/CulturaX", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "6.3 T tokens across 167 languages", "sizeTier": "large", "licence": "ODC-By (plus mC4 and OSCAR terms)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "pretraining", "cleaned", "deduplicated", "mc4", "oscar" ], "description": "A multilingual text dataset of 6.3 T tokens across 167 languages, built for training large language models. It combines cleaned versions of mC4 and OSCAR, two large web corpora, after a multi-stage filtering pipeline.", "links": {} }, { "slug": "data-europa-eu", "name": "data.europa.eu", "page": "https://rag-repo.org/source/data-europa-eu", "url": "https://data.europa.eu", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "web-portal", "api" ], "ragReadiness": "raw", "format": [ "CSV", "JSON", "XML", "RDF" ], "size": "1.6 million+ datasets", "sizeTier": "unknown", "licence": "CC BY 4.0 (mostly)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "eu", "europe", "government", "open-data", "multilingual" ], "description": "The European Data Portal, a single point of access to open data from 36 European countries and the EU institutions. Over 1.6 million datasets span every policy area, giving RAG systems broad, multilingual coverage of official European data.", "links": {} }, { "slug": "data-gov", "name": "data.gov", "page": "https://rag-repo.org/source/data-gov", "url": "https://data.gov", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "web-portal", "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "CSV", "JSON", "XML" ], "size": "300,000+ datasets", "sizeTier": "unknown", "licence": "Public domain (US Government works), varies by dataset", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "US Government", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "us", "government", "open-data", "federal", "public-sector" ], "description": "The US federal government's open data portal, gathering more than 300,000 datasets from federal agencies. Coverage spans agriculture, climate, education, energy, finance, health, and public safety, making it a broad source of official US data for RAG systems.", "links": {} }, { "slug": "data-gov-uk", "name": "data.gov.uk", "page": "https://rag-repo.org/source/data-gov-uk", "url": "https://data.gov.uk", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "web-portal", "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "CSV", "JSON", "XML" ], "size": "47,000+ datasets", "sizeTier": "unknown", "licence": "Open Government Licence v3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Government Digital Service", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "government", "open-data", "statistics", "public-sector" ], "description": "The UK Government's central catalogue of open public sector data, run by the Government Digital Service. More than 47,000 datasets cover health, housing, transport, demographics, and the environment, giving RAG systems a trusted base of UK facts, figures, and official records.", "links": {} }, { "slug": "data-police-uk", "name": "data.police.uk", "page": "https://rag-repo.org/source/data-police-uk", "url": "https://data.police.uk", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "CSV", "JSON" ], "size": null, "sizeTier": "small", "licence": "Open Government Licence v3.0", "licenceUrl": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "UK Home Office", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "crime", "policing", "open-government-licence", "api" ], "description": "Crime and policing data published on behalf of police forces in England and Wales, Northern Ireland, and the British Transport Police, gathered in one central place. Covers street-level crime, outcomes, and stop-and-search records.", "links": { "api": "https://data.police.uk/docs/" } }, { "slug": "datacomp", "name": "DataComp", "page": "https://rag-repo.org/source/datacomp", "url": "https://www.datacomp.ai", "category": "multimodal", "categoryName": "Multimodal & image-text", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "mixed", "format": [], "size": null, "sizeTier": "large", "licence": "CC BY 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "image-text", "benchmark", "data-curation", "web-scale", "multimodal" ], "description": "A benchmark and dataset collection for training multimodal models. It provides a shared pool of image-text candidates and an evaluation framework so teams can test different data-curation strategies against a common yardstick.", "links": {} }, { "slug": "datahub-io", "name": "Datahub.io", "page": "https://rag-repo.org/source/datahub-io", "url": "https://datahub.io", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "open", "accessPatterns": [ "web-portal", "git-repo", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "CSV", "JSON" ], "size": null, "sizeTier": "tiny", "licence": "Varies by dataset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Datopian", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "data-platform", "open-data", "data-packages", "standardised", "csv" ], "description": "A platform for publishing and finding open data packages: datasets bundled with consistent metadata in standardised formats. It hosts curated collections of widely used reference data, from country codes to exchange rates, ready to drop into a pipeline.", "links": {} }, { "slug": "datakind-uk-open-data", "name": "DataKind UK Open Data Sets", "page": "https://rag-repo.org/source/datakind-uk-open-data", "url": "https://www.datakind.org.uk/resources/open-data", "category": "curated-lists", "categoryName": "Curated lists & meta-resources", "accessType": "open", "accessPatterns": [ "web-crawl" ], "ragReadiness": "raw", "format": [ "Markdown" ], "size": null, "sizeTier": "tiny", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "DataKind UK", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "directory", "uk", "open-data", "social-good", "government" ], "description": "A curated list of UK-focused open datasets from DataKind UK, covering government, health, crime, housing, and social data. A quick way into British public data when you are building a RAG system with a UK focus.", "links": {} }, { "slug": "dbpedia", "name": "DBpedia", "page": "https://rag-repo.org/source/dbpedia", "url": "https://www.dbpedia.org", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "mixed", "format": [ "RDF" ], "size": null, "sizeTier": "medium", "licence": "CC BY-SA 3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": 2007, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "knowledge-graph", "structured", "wikipedia", "rdf", "sparql", "multilingual" ], "description": "A knowledge graph built by pulling the structured parts of Wikipedia, mainly the infoboxes, into machine-readable data. It holds billions of facts about people, places, organisations, and more as RDF triples, small subject, predicate, object statements, which you can search with the SPARQL query language.", "links": { "download": "https://www.dbpedia.org/resources/latest-core/" } }, { "slug": "dclm-baseline", "name": "DCLM-Baseline (DataComp-LM)", "page": "https://rag-repo.org/source/dclm-baseline", "url": "https://huggingface.co/datasets/mlfoundations/dclm-baseline-1.0", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": null, "sizeTier": "large", "licence": "ODC-By 1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "DataComp-LM (ML Foundations)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "english", "pretraining", "cleaned", "quality-filtering", "benchmark" ], "description": "A filtered English web dataset from the DataComp-LM benchmark project, produced by running model-based quality filtering over Common Crawl. Built to show which data-curation choices most improve language model training.", "links": { "huggingface": "https://huggingface.co/datasets/mlfoundations/dclm-baseline-1.0" } }, { "slug": "devdocs", "name": "DevDocs", "page": "https://rag-repo.org/source/devdocs", "url": "https://devdocs.io", "category": "code", "categoryName": "Code & technical documentation", "accessType": "open", "accessPatterns": [ "git-repo", "bulk-download" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "small", "licence": "MPL 2.0 (aggregator); individual docs vary", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "DevDocs", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "api-documentation", "developer", "aggregator", "reference", "open-source" ], "description": "An open-source aggregator that pulls API documentation for major programming languages, frameworks, and tools into one searchable place. A tidy RAG source when you want up-to-date developer reference material without scraping dozens of separate documentation sites yourself.", "links": { "github": "https://github.com/freeCodeCamp/devdocs" } }, { "slug": "dpla", "name": "Digital Public Library of America (DPLA)", "page": "https://rag-repo.org/source/dpla", "url": "https://dp.la", "category": "cultural-heritage", "categoryName": "Cultural heritage & archives", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": null, "sizeTier": "small", "licence": "CC0 (metadata)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Digital Public Library of America", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "cultural-heritage", "us", "museums", "libraries", "archives", "metadata" ], "description": "An aggregator that brings together content from America's museums, libraries, and archives. Its API offers metadata on individual items and on collections, and the whole repository is available to download as zipped JSON files.", "links": {} }, { "slug": "dolma", "name": "Dolma", "page": "https://rag-repo.org/source/dolma", "url": "https://huggingface.co/datasets/allenai/dolma", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "JSONL" ], "size": "3T tokens across 4B+ documents", "sizeTier": "large", "licence": "ODC-By 1.0 / AI2 ImpACT License", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Allen Institute for AI (Allen AI)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "pretraining", "multi-source", "open-corpus", "allen-ai" ], "description": "A 3T token open corpus from Allen AI (its name stands for \"Data for Open Language Models' Appetite\") combining web text, scientific papers, code, public-domain books, Reddit posts, and Wikipedia. Built to train the OLMo models.", "links": { "huggingface": "https://huggingface.co/datasets/allenai/dolma" } }, { "slug": "drugbank", "name": "DrugBank", "page": "https://rag-repo.org/source/drugbank", "url": "https://go.drugbank.com", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "limited", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [ "XML", "CSV" ], "size": "15,000+ drug entries", "sizeTier": "tiny", "licence": "CC BY-NC 4.0 (academic); commercial licence available", "licenceUrl": null, "commercialUse": "prohibited", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "OMx Personal Health Analytics", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "drugs", "pharmacology", "drug-targets", "structured", "medical-research" ], "description": "A freely accessible database that combines detailed drug data with drug target information, covering more than 15,000 drug entries. It links chemistry, pharmacology, and biology in one place, which makes it a strong knowledge source for medical and pharmaceutical RAG.", "links": {} }, { "slug": "ecfr", "name": "eCFR", "page": "https://rag-repo.org/source/ecfr", "url": "https://www.ecfr.gov", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "small", "licence": "Public domain (US Government)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "US Government Publishing Office", "updateFrequency": "Continuous", "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "us", "regulations", "public-domain", "api", "bulk-download" ], "description": "An up-to-date electronic Code of Federal Regulations with a full bulk API. Continuously updated, unlike the annual print edition.", "links": {} }, { "slug": "edgar", "name": "EDGAR (SEC Filings)", "page": "https://rag-repo.org/source/edgar", "url": "https://www.sec.gov/edgar/", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "HTML", "XML", "JSON" ], "size": null, "sizeTier": "large", "licence": "Public domain (US Government)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "US Securities and Exchange Commission", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "us", "financial-filings", "sec", "regulatory", "public-domain" ], "description": "The US Securities and Exchange Commission's repository of corporate filings, including annual reports (10-K), quarterly reports (10-Q), current reports (8-K), and proxy statements. You can search the full text or pull filings in bulk through an API, which makes it a rich source for financial and regulatory RAG.", "links": { "api": "https://www.sec.gov/edgar/sec-api-documentation" } }, { "slug": "emilia", "name": "Emilia", "page": "https://rag-repo.org/source/emilia", "url": "https://huggingface.co/datasets/amphion/Emilia-Dataset", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "limited", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "MP3", "JSON" ], "size": "101,000+ hours (216,000+ in Emilia-Large)", "sizeTier": "large", "licence": "CC BY-NC 4.0", "licenceUrl": "https://creativecommons.org/licenses/by-nc/4.0/", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Amphion", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "speech", "text-to-speech", "multilingual", "in-the-wild", "audio", "non-commercial" ], "description": "A large-scale, multilingual in-the-wild speech dataset for speech generation, starting at over 101,000 hours across six languages and extended to more than 216,000 hours in Emilia-Large. Built for training natural, spontaneous TTS and speech-understanding models.", "links": { "documentation": "https://github.com/open-mmlab/Amphion/tree/main/preprocessors/Emilia", "github": "https://github.com/open-mmlab/Amphion", "huggingface": "https://huggingface.co/datasets/amphion/Emilia-Dataset" } }, { "slug": "epo-espacenet", "name": "EPO Espacenet & Open Patent Services", "page": "https://rag-repo.org/source/epo-espacenet", "url": "https://worldwide.espacenet.com", "category": "patents", "categoryName": "Patents & intellectual property", "accessType": "limited", "accessPatterns": [ "api", "web-portal" ], "ragReadiness": "raw", "format": [], "size": "140M+ documents", "sizeTier": "unknown", "licence": "EPO terms (free for most uses)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "European Patent Office", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "patents", "europe", "epo", "api", "full-text", "global" ], "description": "Free access to over 140M patent documents from the European Patent Office (EPO). Its Open Patent Services text analysis tools have made patent full texts much easier to reach, though the terms are open access rather than an open reuse licence.", "links": { "api": "https://worldwide.espacenet.com" } }, { "slug": "essential-web", "name": "Essential-Web v1.0", "page": "https://rag-repo.org/source/essential-web", "url": "https://huggingface.co/datasets/EssentialAI/essential-web-v1.0", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface", "cloud-bucket" ], "ragReadiness": "mixed", "format": [ "Parquet" ], "size": "75.3 TB (24T tokens, ~23.6B documents)", "sizeTier": "huge", "licence": "ODC-BY 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Essential AI", "updateFrequency": "Archived", "yearLaunched": 2025, "isArchived": false, "version": "v1.0", "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "pretraining", "metadata", "filtered", "huggingface", "quality-labels" ], "description": "A 24-trillion-token web corpus (about 23.6 billion documents) from Essential AI where every document carries a twelve-category taxonomy covering topic, format, complexity and quality. The labels let you carve out domain or quality subsets with simple filters, without training your own classifiers.", "links": { "documentation": "https://arxiv.org/abs/2506.14111", "huggingface": "https://huggingface.co/datasets/EssentialAI/essential-web-v1.0" } }, { "slug": "eur-lex", "name": "EUR-Lex", "page": "https://rag-repo.org/source/eur-lex", "url": "https://eur-lex.europa.eu", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "XML", "HTML", "RDF" ], "size": null, "sizeTier": "unknown", "licence": "Reuse authorised (Commission Decision 2011/833/EU)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Publications Office of the European Union", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "eu", "legislation", "case-law", "multilingual", "regulatory", "treaties" ], "description": "The official portal for European Union law. It provides EU treaties, legislation, case law, and legislative proposals in 24 languages, with bulk download available. A dependable source for multilingual legal and regulatory RAG across the EU.", "links": {} }, { "slug": "europeana", "name": "Europeana", "page": "https://rag-repo.org/source/europeana", "url": "https://www.europeana.eu", "category": "cultural-heritage", "categoryName": "Cultural heritage & archives", "accessType": "open", "accessPatterns": [ "api", "web-portal" ], "ragReadiness": "cleaned", "format": [], "size": "60M+ items", "sizeTier": "unknown", "licence": "CC0 (metadata); varies by item (content)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Europeana Foundation", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "cultural-heritage", "europe", "multilingual", "museums", "libraries", "metadata" ], "description": "A platform for discovering cultural heritage collections across Europe, with multilingual access to over 60 million digitised items from around 4,000 institutions, including books, paintings, maps, manuscripts, and audiovisual and 3D media.", "links": {} }, { "slug": "eurostat", "name": "Eurostat", "page": "https://rag-repo.org/source/eurostat", "url": "https://ec.europa.eu/eurostat", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "unknown", "licence": "Reuse authorised (Commission Decision 2011/833/EU)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "European Commission", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "statistics", "eu", "europe", "economics", "official-statistics", "api" ], "description": "The statistical office of the European Union, providing official statistics for Europe. Covers the economy, population, environment, agriculture, trade, and more across EU member states, with harmonised figures that let you compare countries on a like-for-like basis.", "links": {} }, { "slug": "financebench", "name": "FinanceBench", "page": "https://rag-repo.org/source/financebench", "url": "https://github.com/patronus-ai/financebench", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "limited", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "cleaned", "format": [ "JSONL", "PDF" ], "size": "10,231 QA triplets (150 open sample)", "sizeTier": "tiny", "licence": "CC BY-NC 4.0", "licenceUrl": "https://creativecommons.org/licenses/by-nc/4.0/", "commercialUse": "prohibited", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Patronus AI", "updateFrequency": null, "yearLaunched": 2023, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "finance", "question-answering", "evaluation", "sec-filings", "retrieval-benchmark" ], "description": "A benchmark for evaluating LLMs and retrieval systems on questions about public financial filings. The full set holds 10,231 question, answer and evidence triplets; a 150 example sample is released openly.", "links": { "download": "https://huggingface.co/datasets/PatronusAI/financebench", "github": "https://github.com/patronus-ai/financebench", "huggingface": "https://huggingface.co/datasets/PatronusAI/financebench" } }, { "slug": "find-case-law", "name": "Find Case Law (UK National Archives)", "page": "https://rag-repo.org/source/find-case-law", "url": "https://caselaw.nationalarchives.gov.uk", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "XML" ], "size": null, "sizeTier": "unknown", "licence": "Open Justice Licence", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "The National Archives (UK)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "case-law", "uk", "court-opinions", "api", "legal" ], "description": "Official UK court decisions published by the National Archives, with structured XML and a documented API. The authoritative modern source for downloading UK judgments.", "links": {} }, { "slug": "finepdfs", "name": "FinePDFs", "page": "https://rag-repo.org/source/finepdfs", "url": "https://huggingface.co/datasets/HuggingFaceFW/finepdfs", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~3.65 TB (3T tokens)", "sizeTier": "large", "licence": "ODC-BY 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "HuggingFace", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "pdf", "multilingual", "pretraining", "common-crawl", "huggingface", "filtered", "long-form" ], "description": "HuggingFace's corpus built entirely from PDFs: roughly 3 trillion tokens across 475 million documents in 1,733 languages, drawn from Common Crawl and the open web. It captures dense long-form content (reports, manuals, papers) that HTML web corpora miss, and is the PDF counterpart to FineWeb.", "links": { "huggingface": "https://huggingface.co/datasets/HuggingFaceFW/finepdfs" } }, { "slug": "fineweb", "name": "FineWeb / FineWeb-Edu", "page": "https://rag-repo.org/source/fineweb", "url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "45 TB (15T tokens)", "sizeTier": "large", "licence": "ODC-By 1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "HuggingFace", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "english", "pretraining", "filtered", "deduplicated", "huggingface" ], "description": "A 15T token English web corpus distilled from Common Crawl by HuggingFace, filtered and deduplicated for language model training. FineWeb-Edu is a subset filtered for educational content. One of the higher-quality open web datasets for pretraining and broad-coverage RAG.", "links": { "huggingface": "https://huggingface.co/datasets/HuggingFaceFW/fineweb" } }, { "slug": "fineweb-edu", "name": "FineWeb-Edu", "page": "https://rag-repo.org/source/fineweb-edu", "url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~1.3T tokens (5.4T token score>=2 variant)", "sizeTier": "large", "licence": "ODC-By 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "HuggingFace", "updateFrequency": "Archived", "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "english", "pretraining", "filtered", "educational", "huggingface" ], "description": "The educational-quality subset of FineWeb, about 1.3 trillion English tokens kept by an educational-quality classifier, with a larger 5.4T token variant at a looser threshold. Built by HuggingFace as a denser, more teachable base for pretraining and broad-coverage RAG.", "links": { "huggingface": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu" } }, { "slug": "fineweb2", "name": "FineWeb2", "page": "https://rag-repo.org/source/fineweb2", "url": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-2", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": null, "sizeTier": "large", "licence": "ODC-By 1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "HuggingFace", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "2", "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "multilingual", "pretraining", "filtered", "deduplicated", "huggingface" ], "description": "A multilingual extension of FineWeb from HuggingFace, built with an open curation pipeline that adapts filtering and deduplication across languages. Useful when you need clean web text beyond English for pretraining or multilingual RAG.", "links": { "huggingface": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-2" } }, { "slug": "fineweb2-hq", "name": "FineWeb2-HQ", "page": "https://rag-repo.org/source/fineweb2-hq", "url": "https://huggingface.co/datasets/epfml/FineWeb2-HQ", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~6.3 TB (about 380M documents)", "sizeTier": "large", "licence": "ODC-By 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "EPFL Machine Learning and Optimization Laboratory", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": "2-HQ", "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "pretraining", "web-crawl", "quality-filtered", "large-corpus" ], "description": "A high-quality multilingual pretraining corpus of roughly the top 10% of FineWeb2 documents in each of 20 languages, selected by a model-based quality classifier. Built by EPFL and released on Hugging Face.", "links": { "github": "https://github.com/epfml/fineweb2-hq", "huggingface": "https://huggingface.co/datasets/epfml/FineWeb2-HQ" } }, { "slug": "fingertips", "name": "Fingertips (Public Health Data)", "page": "https://rag-repo.org/source/fingertips", "url": "https://fingertips.phe.org.uk", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "CSV" ], "size": null, "sizeTier": "small", "licence": "Open Government Licence v3.0", "licenceUrl": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Office for Health Improvement and Disparities", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "public-health", "statistics", "mental-health", "open-government-licence" ], "description": "A large public health data collection for England, including mental health indicators and the Public Health Outcomes Framework. Useful for viewing trends over time and comparing differences between areas.", "links": {} }, { "slug": "flashrag", "name": "FlashRAG Benchmark Datasets", "page": "https://rag-repo.org/source/flashrag", "url": "https://github.com/RUC-NLPIR/FlashRAG", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "mixed", "format": [ "JSONL" ], "size": null, "sizeTier": "small", "licence": "MIT (code); varies (datasets)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "RUC-NLPIR", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "evaluation", "benchmark", "toolkit", "jsonl", "question-answering", "research" ], "description": "A research toolkit that bundles standardised, pre-processed benchmark datasets for RAG evaluation across many scenarios. Every dataset comes in a single unified JSONL format, so you can swap benchmarks in and out without rewriting your data loading each time.", "links": { "github": "https://github.com/RUC-NLPIR/FlashRAG" } }, { "slug": "frames-benchmark", "name": "FRAMES (Fact, Fetch, and Reason)", "page": "https://rag-repo.org/source/frames-benchmark", "url": "https://huggingface.co/datasets/google/frames-benchmark", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "CSV", "Parquet" ], "size": "824 items", "sizeTier": "tiny", "licence": "Apache 2.0", "licenceUrl": "https://www.apache.org/licenses/LICENSE-2.0", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Google", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "rag-evaluation", "factuality", "multi-hop", "reasoning", "question-answering", "wikipedia", "benchmark" ], "description": "A Google evaluation set of 824 challenging questions that tests factuality, multi-hop retrieval and reasoning together, each paired with a gold answer and the Wikipedia articles needed to answer it. Built to measure end-to-end RAG.", "links": { "huggingface": "https://huggingface.co/datasets/google/frames-benchmark" } }, { "slug": "fred", "name": "FRED (Federal Reserve Economic Data)", "page": "https://rag-repo.org/source/fred", "url": "https://fred.stlouisfed.org/", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "JSON", "XML", "CSV" ], "size": "~845,000 series", "sizeTier": "unknown", "licence": "Varies (see FRED legal notices)", "licenceUrl": "https://fred.stlouisfed.org/legal", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Federal Reserve Bank of St. Louis", "updateFrequency": null, "yearLaunched": 1991, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "economics", "time-series", "macroeconomics", "api", "us", "government" ], "description": "A large online database of macroeconomic and financial time series maintained by the Federal Reserve Bank of St. Louis, with around 845,000 series drawn from over 120 public and private sources. Free to access through a REST API returning JSON or XML.", "links": { "documentation": "https://fred.stlouisfed.org/docs/api/fred/", "api": "https://fred.stlouisfed.org/docs/api/fred/" } }, { "slug": "courtlistener", "name": "Free Law Project / CourtListener", "page": "https://rag-repo.org/source/courtlistener", "url": "https://www.courtlistener.com", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "JSON", "CSV", "XML" ], "size": null, "sizeTier": "large", "licence": "Public domain (US court opinions)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Free Law Project", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "legal", "us", "case-law", "court-opinions", "public-domain", "nonprofit" ], "description": "The largest open collection of US court opinions, oral argument recordings, and judicial financial disclosures, run by the nonprofit Free Law Project. Its RECAP Archive holds hundreds of millions of federal court docket entries, and journalists at the Wall Street Journal and ProPublica have used it for investigations. A solid base for US legal RAG.", "links": { "api": "https://www.courtlistener.com/help/api/" } }, { "slug": "freebase", "name": "Freebase", "page": "https://rag-repo.org/source/freebase", "url": "https://developers.google.com/freebase", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "mixed", "format": [ "RDF" ], "size": null, "sizeTier": "small", "licence": "CC BY 2.5", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Google (archived)", "updateFrequency": "Archived", "yearLaunched": 2007, "isArchived": true, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "knowledge-base", "structured", "archived", "google", "rdf" ], "description": "A collaborative knowledge base once run by Google, now retired but still available as downloadable data dumps. It holds structured facts about millions of entities, and much of its content has since moved into Wikidata.", "links": { "download": "https://developers.google.com/freebase#freebase-rdf-dumps" } }, { "slug": "freshstack", "name": "FreshStack", "page": "https://rag-repo.org/source/freshstack", "url": "https://fresh-stack.github.io/", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "chunked", "format": [ "JSON", "JSONL" ], "size": null, "sizeTier": "small", "licence": "Datasets CC BY-SA 4.0; evaluation code Apache 2.0", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "University of Waterloo and Databricks", "updateFrequency": "Archived", "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "code", "stack-overflow", "evaluation", "technical-docs" ], "description": "A 2025 framework and benchmark for retrieval over fast-moving technical documentation. It pairs real Stack Overflow questions and answers with chunked GitHub code and docs across five niche domains, giving code and docs RAG a hard, contamination-resistant test.", "links": { "github": "https://github.com/fresh-stack/freshstack", "huggingface": "https://huggingface.co/freshstack" } }, { "slug": "gdelt", "name": "GDELT", "page": "https://rag-repo.org/source/gdelt", "url": "https://www.gdeltproject.org", "category": "news-media", "categoryName": "News, events & media", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "CSV" ], "size": "2.5+ TB per year", "sizeTier": "large", "licence": "Open (free for research and commercial use)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": "Every 15 minutes", "yearLaunched": 2013, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "news", "events", "multilingual", "sentiment", "knowledge-graph", "open-data" ], "description": "GDELT, the Global Database of Events, Language, and Tone, monitors print, broadcast, and web news in more than 100 languages from every country. It records events back to 1979, builds a Global Knowledge Graph, and adds sentiment and emotion analysis, all queryable on Google BigQuery.", "links": { "documentation": "https://www.gdeltproject.org/data.html" } }, { "slug": "geonames", "name": "GeoNames", "page": "https://rag-repo.org/source/geonames", "url": "https://www.geonames.org", "category": "geospatial", "categoryName": "Geospatial & mapping", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "TSV", "JSON", "XML" ], "size": "11M+ place names", "sizeTier": "small", "licence": "CC BY 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": "Daily", "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "gazetteer", "place-names", "coordinates", "api", "structured", "geographic" ], "description": "A geographical database of more than 11 million place names, each with coordinates, population, elevation, and administrative divisions, covering every country. It works well as a gazetteer for resolving place names to locations.", "links": { "api": "https://www.geonames.org/export/web-services.html", "download": "https://download.geonames.org/export/dump/" } }, { "slug": "github-archive", "name": "GitHub Public Repositories (GH Archive / GHTorrent)", "page": "https://rag-repo.org/source/github-archive", "url": "https://www.gharchive.org", "category": "code", "categoryName": "Code & technical documentation", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "JSON" ], "size": null, "sizeTier": "large", "licence": "GitHub ToS; content under original repo licences", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": null, "updateFrequency": null, "yearLaunched": 2012, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "github", "event-data", "issues", "pull-requests", "metadata", "mining" ], "description": "Two projects that capture public GitHub activity for analysis. GH Archive records the public GitHub event timeline as downloadable hourly archives, and GHTorrent offers a queryable mirror of GitHub metadata. Together they help you mine code, issues, pull requests, and documentation at scale.", "links": { "documentation": "https://www.gharchive.org" } }, { "slug": "glaive-function-calling", "name": "Glaive Function Calling v2", "page": "https://rag-repo.org/source/glaive-function-calling", "url": "https://huggingface.co/datasets/glaiveai/glaive-function-calling-v2", "category": "agentic-tool-use", "categoryName": "Agentic & Tool-Use", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSON", "Parquet" ], "size": "~113,000 examples", "sizeTier": "tiny", "licence": "Apache 2.0", "licenceUrl": "https://www.apache.org/licenses/LICENSE-2.0", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Glaive AI", "updateFrequency": null, "yearLaunched": 2023, "isArchived": false, "version": "v2", "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "function-calling", "tool-use", "synthetic", "fine-tuning", "multi-turn", "sft", "chat" ], "description": "A widely used open dataset of about 113,000 synthetic multi-turn chat conversations that include function calls and their results, made by Glaive AI. One of the most downloaded open datasets for fine-tuning models to call tools, released under Apache 2.0.", "links": { "huggingface": "https://huggingface.co/datasets/glaiveai/glaive-function-calling-v2" } }, { "slug": "glotcc", "name": "GlotCC", "page": "https://rag-repo.org/source/glotcc", "url": "https://huggingface.co/datasets/cis-lmu/GlotCC-V1", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": null, "sizeTier": "large", "licence": "CC0 (pipeline); content per Common Crawl terms", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "low-resource", "minority-languages", "common-crawl", "cleaned" ], "description": "An open, broad-coverage corpus and processing pipeline built from Common Crawl, targeting minority and low-resource languages that larger corpora tend to miss. It fills gaps left by datasets focused on higher-resource languages.", "links": {} }, { "slug": "google-books-ngrams", "name": "Google Books Ngrams", "page": "https://rag-repo.org/source/google-books-ngrams", "url": "https://books.google.com/ngrams", "category": "books-literature", "categoryName": "Books & literature", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [ "TSV" ], "size": "~2.2 TB", "sizeTier": "large", "licence": "CC BY 3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Google", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "ngrams", "word-frequency", "historical", "linguistics", "english", "multilingual" ], "description": "Word and phrase frequency data drawn from Google's digitised book collection, spanning centuries of published text. It counts how often words and short phrases appear by year, which is useful for historical language analysis and time-aware RAG features.", "links": {} }, { "slug": "google-dataset-search", "name": "Google Dataset Search", "page": "https://rag-repo.org/source/google-dataset-search", "url": "https://datasetsearch.research.google.com", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "open", "accessPatterns": [ "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Google", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "data-platform", "search-engine", "discovery", "meta-resource", "google" ], "description": "A search engine for datasets that indexes millions of them from thousands of repositories worldwide. It does not host data itself: it points you to wherever each dataset lives, which makes it a fast first stop when you are hunting for a source on a specific topic.", "links": {} }, { "slug": "google-knowledge-graph", "name": "Google Knowledge Graph API", "page": "https://rag-repo.org/source/google-knowledge-graph", "url": "https://developers.google.com/knowledge-graph", "category": "knowledge-graphs", "categoryName": "Knowledge graphs & structured data", "accessType": "limited", "accessPatterns": [ "api" ], "ragReadiness": "mixed", "format": [ "JSON" ], "size": null, "sizeTier": "unknown", "licence": "Google API Terms", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Google", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "api", "knowledge-graph", "entities", "google", "freemium" ], "description": "An API into Google's Knowledge Graph, the store of billions of facts about entities that powers the info panels you see beside search results. You send a name or query and get back matching entities with descriptions, types, and links. Free to use within rate limits.", "links": { "documentation": "https://developers.google.com/knowledge-graph/reference/rest/v1", "api": "https://developers.google.com/knowledge-graph" } }, { "slug": "google-patents-public", "name": "Google Patents Public Datasets", "page": "https://rag-repo.org/source/google-patents-public", "url": "https://console.cloud.google.com/marketplace/details/google_patents_public_datasets/google-patents-public-data", "category": "patents", "categoryName": "Patents & intellectual property", "accessType": "open", "accessPatterns": [ "api" ], "ragReadiness": "cleaned", "format": [ "BigQuery" ], "size": "120M+ documents", "sizeTier": "large", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Google", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "patents", "multilingual", "bigquery", "structured", "global" ], "description": "A query-based collection of over 120M patent documents from more than 100 patent offices worldwide, including applications, pre-grant publications, and granted patents. Accessible through Google BigQuery, with hundreds of millions of USPTO events also queryable.", "links": {} }, { "slug": "alphaearth-satellite-embedding", "name": "Google Satellite Embedding (AlphaEarth Foundations)", "page": "https://rag-repo.org/source/alphaearth-satellite-embedding", "url": "https://developers.google.com/earth-engine/datasets/catalog/GOOGLE_SATELLITE_EMBEDDING_V1_ANNUAL", "category": "embedded-datasets", "categoryName": "Pre-embedded & RAG-ready", "accessType": "open", "accessPatterns": [ "api", "cloud-bucket", "bulk-download" ], "ragReadiness": "embedded", "format": [ "Earth Engine Image Collection", "GeoTIFF" ], "size": "~1.4 T+ embedding footprints per year", "sizeTier": "huge", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Google and Google DeepMind", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "geospatial", "satellite", "embeddings", "remote-sensing", "earth-observation", "google" ], "description": "Global annual satellite embeddings produced by Google and Google DeepMind's AlphaEarth Foundations model. Every 10 metre pixel holds a 64 dimensional vector summarising a year of multi-sensor Earth observation, available yearly from 2017.", "links": { "documentation": "https://developers.google.com/earth-engine/tutorials/community/satellite-embedding-01-introduction" } }, { "slug": "google-microsoft-open-buildings", "name": "Google-Microsoft-OSM Open Buildings (VIDA)", "page": "https://rag-repo.org/source/google-microsoft-open-buildings", "url": "https://source.coop/vida/google-microsoft-osm-open-buildings", "category": "geospatial", "categoryName": "Geospatial & mapping", "accessType": "open", "accessPatterns": [ "cloud-bucket", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "GeoParquet", "FlatGeobuf", "PMTiles" ], "size": "Roughly 2.7 billion building footprints (2,705,459,584)", "sizeTier": "medium", "licence": "Open Data Commons Open Database Licence (ODbL) v1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "VIDA (hosted on Source Cooperative, a Radiant Earth project)", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "building-footprints", "geospatial", "global", "satellite-imagery", "open-data", "cloud-native", "share-alike" ], "description": "A conflated global building-footprint layer that merges Google Open Buildings V3, Microsoft Global ML Footprints, and OpenStreetMap into roughly 2.7 billion footprints, each labelled by its source. Built by VIDA and hosted on Source Cooperative in cloud-native formats, it offers more complete coverage than any single provider on its own.", "links": { "documentation": "https://medium.com/vida-engineering/updating-the-ultimate-cloud-native-building-footprints-dataset-6d4384cb93c4" } }, { "slug": "govinfo", "name": "GovInfo", "page": "https://rag-repo.org/source/govinfo", "url": "https://www.govinfo.gov", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "medium", "licence": "Public domain (US Government)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "US Government Publishing Office", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "us", "legislation", "regulations", "public-domain", "api", "bulk-download" ], "description": "US federal legislation, regulations, and congressional records, with bulk data and API access. The authoritative source for US federal government publications.", "links": {} }, { "slug": "granary", "name": "Granary", "page": "https://rag-repo.org/source/granary", "url": "https://huggingface.co/datasets/nvidia/Granary", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSON", "Parquet" ], "size": "~1M hours (25 languages)", "sizeTier": "huge", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "NVIDIA", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "speech", "asr", "speech-translation", "multilingual", "european-languages", "audio" ], "description": "An open multilingual speech corpus from NVIDIA covering roughly one million hours of audio across 25 European languages, built for automatic speech recognition and speech translation.", "links": { "huggingface": "https://huggingface.co/datasets/nvidia/Granary" } }, { "slug": "hansard", "name": "Hansard", "page": "https://rag-repo.org/source/hansard", "url": "https://hansard.parliament.uk", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "XML" ], "size": null, "sizeTier": "small", "licence": "Open Parliament Licence", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "UK Parliament", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "parliament", "government", "transcripts", "linked-data" ], "description": "The official word-for-word record of proceedings in the UK Parliament, both the Commons and the Lords. Full text searchable and available as XML and linked data going back centuries, it is a rich source of political and legislative debate for RAG systems.", "links": {} }, { "slug": "hupd", "name": "Harvard USPTO Patent Dataset (HUPD)", "page": "https://rag-repo.org/source/hupd", "url": "https://huggingface.co/datasets/HUPD/hupd", "category": "patents", "categoryName": "Patents & intellectual property", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet", "JSON" ], "size": null, "sizeTier": "small", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Harvard", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "patents", "us", "uspto", "nlp", "machine-learning", "research" ], "description": "A large-scale, structured corpus of US patent applications built specifically for machine learning and natural language processing research. It fills the gap left by mainstream patent search tools, which are not designed with the ML and NLP community in mind.", "links": { "huggingface": "https://huggingface.co/datasets/HUPD/hupd" } }, { "slug": "hathitrust", "name": "HathiTrust Digital Library", "page": "https://rag-repo.org/source/hathitrust", "url": "https://www.hathitrust.org", "category": "cultural-heritage", "categoryName": "Cultural heritage & archives", "accessType": "limited", "accessPatterns": [ "bulk-download", "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "large", "licence": "Varies; non-consumptive research access for in-copyright material", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "HathiTrust", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "digitised-books", "academic", "libraries", "research-access", "text-mining" ], "description": "A partnership of academic and research institutions offering millions of digitised titles. Its Research Center provides computational access to the full corpus, including in-copyright works, under non-consumptive research terms.", "links": {} }, { "slug": "hplt", "name": "HPLT (High Performance Language Technologies)", "page": "https://rag-repo.org/source/hplt", "url": "https://hplt-project.org", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": "75 languages", "sizeTier": "large", "licence": "CC0 (where possible)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "HPLT project (EU-funded consortium)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "web-crawl", "internet-archive", "eu", "cc0" ], "description": "An EU-funded project building open language resources from web crawls and the Internet Archive. Its MonoHPLT dataset contains monolingual text in 75 languages, released as openly as licensing allows.", "links": {} }, { "slug": "huggingface-datasets", "name": "Hugging Face Datasets", "page": "https://rag-repo.org/source/huggingface-datasets", "url": "https://huggingface.co/datasets", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "open", "accessPatterns": [ "huggingface", "web-portal" ], "ragReadiness": "mixed", "format": [ "Parquet", "JSON", "CSV", "Arrow" ], "size": null, "sizeTier": "unknown", "licence": "Varies by dataset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Hugging Face", "updateFrequency": "Continuous", "yearLaunched": 2020, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "platform", "machine-learning", "streaming", "versioning", "community" ], "description": "The largest open hub for machine learning datasets, with well over 100,000 datasets you can search, stream, and version. Many RAG-ready corpora live here, and its Python library lets you pull data straight into a pipeline in a few lines.", "links": { "documentation": "https://huggingface.co/docs/datasets" } }, { "slug": "imf-data", "name": "IMF Data", "page": "https://rag-repo.org/source/imf-data", "url": "https://www.imf.org/en/data", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "tiny", "licence": "IMF terms (free for most uses)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "International Monetary Fund", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "statistics", "economics", "finance", "macroeconomics", "global", "api" ], "description": "A gateway to the IMF's global economic data, giving streamlined access to macroeconomic and financial statistics. Includes International Financial Statistics covering balance of payments, interest rates, national accounts, prices, production, trade, and population for more than 200 countries.", "links": {} }, { "slug": "institutional-books", "name": "Institutional Books 1.0", "page": "https://rag-repo.org/source/institutional-books", "url": "https://institutional.org/institutional-books", "category": "books-literature", "categoryName": "Books & literature", "accessType": "limited", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "mixed", "format": [ "Parquet" ], "size": "242 B tokens", "sizeTier": "large", "licence": "Institutional Books Early-Access Terms (noncommercial)", "licenceUrl": "https://huggingface.co/datasets/institutional/institutional-books-1.0", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Institutional Data Initiative (Harvard Law School Library)", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "public-domain", "books", "ocr", "multilingual", "library", "pretraining" ], "description": "A 242 billion token dataset of roughly 983,000 public-domain volumes digitised from Harvard Library's collections, spanning more than 250 languages, with both raw and post-processed OCR text plus rich bibliographic metadata.", "links": { "huggingface": "https://huggingface.co/datasets/institutional/institutional-books-1.0" } }, { "slug": "internet-archive", "name": "Internet Archive", "page": "https://rag-repo.org/source/internet-archive", "url": "https://archive.org", "category": "news-media", "categoryName": "News, events & media", "accessType": "open", "accessPatterns": [ "web-portal", "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "Various" ], "size": "800B+ archived web pages", "sizeTier": "huge", "licence": "Varies by collection", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Internet Archive (nonprofit)", "updateFrequency": null, "yearLaunched": 1996, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "digital-library", "nonprofit", "web-archive", "books", "audio", "television" ], "description": "A nonprofit digital library giving free access to millions of books, films, audio recordings, software, archived websites, and television news. Its Wayback Machine has saved more than 800 billion web pages, making it a deep well of historical and current text.", "links": { "api": "https://archive.org/developers/" } }, { "slug": "kaggle", "name": "Kaggle Datasets", "page": "https://rag-repo.org/source/kaggle", "url": "https://www.kaggle.com/datasets", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "open", "accessPatterns": [ "web-portal", "api" ], "ragReadiness": "mixed", "format": [ "CSV", "JSON", "SQLite" ], "size": null, "sizeTier": "unknown", "licence": "Varies by dataset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Google", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "data-platform", "community", "notebooks", "competitions", "google" ], "description": "A hub of user-contributed datasets spanning almost every domain, owned by Google. Alongside the data you get notebooks, competitions, and active discussion, which makes it a quick way to find something workable and see how others have already cleaned and used it.", "links": {} }, { "slug": "khan-academy", "name": "Khan Academy", "page": "https://rag-repo.org/source/khan-academy", "url": "https://www.khanacademy.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "web-crawl" ], "ragReadiness": "raw", "format": [ "HTML", "Video" ], "size": null, "sizeTier": "unknown", "licence": "CC BY-NC-SA 4.0", "licenceUrl": "https://creativecommons.org/licenses/by-nc-sa/4.0/", "commercialUse": "prohibited", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Khan Academy", "updateFrequency": "Ongoing", "yearLaunched": 2008, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "education", "k-12", "mastery-learning", "curriculum-aligned", "non-commercial", "nonprofit" ], "description": "A free nonprofit learning platform organised as a subject-to-skill knowledge tree across maths, science, economics and the humanities, from primary through early college. Videos, articles and mastery-based exercises, but non-commercial licensing and no public data API.", "links": {} }, { "slug": "laion-400m", "name": "LAION-400M with CLIP Embeddings", "page": "https://rag-repo.org/source/laion-400m", "url": "https://laion.ai/blog/laion-400-open-dataset/", "category": "embedded-datasets", "categoryName": "Pre-embedded & RAG-ready", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "embedded", "format": [], "size": "400 M image-text pairs", "sizeTier": "medium", "licence": "CC BY 4.0 (metadata); images under original terms", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "LAION", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "400M", "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": "laion-5b", "tags": [ "image-text", "clip", "embeddings", "multimodal", "web-crawl", "similarity-search" ], "description": "400 million image-text pairs filtered from Common Crawl using CLIP, a model that scores how well an image matches a caption, and released alongside their pre-computed CLIP embeddings and kNN indices for fast similarity search. The ready-made embeddings make it unusually easy to work with. Note that it ships image URLs, not the images themselves, so some links have decayed.", "links": { "download": "https://laion.ai/blog/laion-400-open-dataset/" } }, { "slug": "laion-5b", "name": "LAION-5B", "page": "https://rag-repo.org/source/laion-5b", "url": "https://laion.ai/blog/laion-5b/", "category": "multimodal", "categoryName": "Multimodal & image-text", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [], "size": "5.8B pairs", "sizeTier": "large", "licence": "CC BY 4.0 (metadata); images under original terms", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "LAION", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "5B", "lastReviewed": "2026-08-06", "supersedes": [ "laion-400m" ], "supersededBy": null, "tags": [ "image-text", "clip-filtered", "web-scale", "multilingual", "url-list", "multimodal" ], "description": "An image-text dataset of over 5.8B examples, built by filtering Common Crawl with a CLIP model that scores how well an image matches a caption. Includes 2.32B English pairs, 2.26B multilingual pairs, and 1.27B not tied to any particular language. Provides URLs, not the images themselves.", "links": {} }, { "slug": "legalbench-rag", "name": "LegalBench-RAG", "page": "https://rag-repo.org/source/legalbench-rag", "url": "https://github.com/zeroentropy-ai/legalbenchrag", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "chunked", "format": [ "JSON", "TXT" ], "size": "6,858 query-answer pairs over a ~79M-character corpus", "sizeTier": "tiny", "licence": "MIT (benchmark code and packaging); component datasets CUAD, MAUD, ContractNLI and PrivacyQA each carry their own licence", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "ZeroEntropy", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "legal", "contracts", "evaluation", "rag-evaluation" ], "description": "The first open benchmark for the retrieval step of legal RAG. It offers 6,858 human-annotated query-answer pairs over a corpus of roughly 79 million characters, with character-level ground-truth spans across contracts and privacy policies, assembled from CUAD, MAUD, ContractNLI and PrivacyQA.", "links": { "documentation": "https://arxiv.org/abs/2408.10343", "github": "https://github.com/zeroentropy-ai/legalbenchrag" } }, { "slug": "legislation-gov-uk", "name": "Legislation.gov.uk", "page": "https://rag-repo.org/source/legislation-gov-uk", "url": "https://www.legislation.gov.uk", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "XML" ], "size": null, "sizeTier": "small", "licence": "Open Government Licence v3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "legislation", "statutes", "linked-data", "government" ], "description": "The official UK government legislation database, holding all UK Acts of Parliament and statutory instruments. It is available as XML and linked data, so you can load the full statute book into a RAG system rather than scraping individual pages.", "links": {} }, { "slug": "lematerial", "name": "LeMaterial (LeMat-Bulk)", "page": "https://rag-repo.org/source/lematerial", "url": "https://huggingface.co/datasets/LeMaterial/LeMat-Bulk", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~6.7M entries", "sizeTier": "small", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "LeMaterial (Entalpic + Hugging Face)", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "materials-science", "chemistry", "computational", "structured", "research", "open" ], "description": "A harmonised, standardised merge of three major computational materials databases (Materials Project, Alexandria and OQMD) into a single format of roughly 6.7 million entries with consistent property definitions.", "links": { "huggingface": "https://huggingface.co/datasets/LeMaterial/LeMat-Bulk" } }, { "slug": "libretexts", "name": "LibreTexts", "page": "https://rag-repo.org/source/libretexts", "url": "https://libretexts.org", "category": "education", "categoryName": "Education & open learning", "accessType": "limited", "accessPatterns": [ "bulk-download", "web-crawl" ], "ragReadiness": "cleaned", "format": [ "HTML", "PDF" ], "size": null, "sizeTier": "small", "licence": "CC BY-NC-SA (mostly)", "licenceUrl": null, "commercialUse": "prohibited", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "LibreTexts", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "education", "open-textbooks", "cc-by-nc-sa", "higher-education", "science" ], "description": "One of the largest open textbook platforms, covering higher education with some K-12 material. Hosts interlinked textbooks and reference material across science, mathematics, humanities, and social sciences. Mostly NonCommercial, so commercial use is ruled out.", "links": {} }, { "slug": "librispeech", "name": "LibriSpeech", "page": "https://rag-repo.org/source/librispeech", "url": "https://www.openslr.org/12", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "cleaned", "format": [ "FLAC", "TXT" ], "size": "1,000 hours", "sizeTier": "medium", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "OpenSLR", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "speech", "english", "asr", "public-domain", "benchmark", "audiobooks" ], "description": "A collection of 1,000 hours of read English speech, released under CC BY 4.0 and stored using the open-source FLAC audio encoder. Labels are aligned at the sentence level. Derived from LibriVox public domain audiobooks, it is the standard benchmark for English automatic speech recognition.", "links": {} }, { "slug": "librivox", "name": "LibriVox", "page": "https://rag-repo.org/source/librivox", "url": "https://librivox.org", "category": "books-literature", "categoryName": "Books & literature", "accessType": "open", "accessPatterns": [ "bulk-download", "web-portal" ], "ragReadiness": "raw", "format": [ "audio", "MP3" ], "size": null, "sizeTier": "large", "licence": "Public domain", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "LibriVox", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "public-domain", "audiobooks", "audio", "speech", "multimodal", "nonprofit" ], "description": "A volunteer project providing free public domain audiobooks. Volunteers record themselves reading works whose copyright has expired, which makes it a handy source of speech data for training speech-to-text models and building multimodal RAG.", "links": {} }, { "slug": "lj-speech", "name": "LJ Speech", "page": "https://rag-repo.org/source/lj-speech", "url": "https://keithito.com/LJ-Speech-Dataset/", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "cleaned", "format": [ "WAV", "CSV" ], "size": "13,100 clips", "sizeTier": "small", "licence": "Public domain", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "speech", "english", "text-to-speech", "public-domain", "single-speaker", "benchmark" ], "description": "An open dataset of 13,100 short audio clips of a single speaker reading passages from seven non-fiction books. Every clip is transcribed and clips run from 1 to 10 seconds. Released into the public domain, it is the standard single-speaker text-to-speech benchmark.", "links": {} }, { "slug": "london-datastore", "name": "London Datastore", "page": "https://rag-repo.org/source/london-datastore", "url": "https://data.london.gov.uk", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "web-portal", "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "CSV", "Excel", "JSON" ], "size": "700+ datasets", "sizeTier": "small", "licence": "Open Government Licence v3.0", "licenceUrl": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Greater London Authority", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "london", "regional", "statistics", "open-government-licence" ], "description": "A free and open data-sharing site for data and analysis from the Greater London Authority, providing more than 700 datasets at regional and local levels across the capital.", "links": {} }, { "slug": "longembed", "name": "LongEmbed", "page": "https://rag-repo.org/source/longembed", "url": "https://github.com/dwzhu-pku/LongEmbed", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "git-repo", "huggingface" ], "ragReadiness": "chunked", "format": [], "size": null, "sizeTier": "tiny", "licence": "MIT", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "long-context", "embeddings", "evaluation" ], "description": "A retrieval benchmark focused on very long documents, with an average length above 5,500 words. It addresses a gap in standard benchmarks, which mostly use short documents of at most 512 tokens (the chunks of text a model reads at once).", "links": {} }, { "slug": "madlad-400", "name": "MADLAD-400", "page": "https://rag-repo.org/source/madlad-400", "url": "https://huggingface.co/datasets/allenai/MADLAD-400", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "mixed", "format": [ "JSON" ], "size": "3 T tokens across 419 languages", "sizeTier": "large", "licence": "ODC-By 1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Google DeepMind and Google Research", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "pretraining", "common-crawl", "google", "low-resource" ], "description": "A manually audited, general-domain monolingual dataset of 3 T tokens spanning 419 languages, built from Common Crawl by Google DeepMind and Google Research. It comes in a noisy version and a more heavily filtered clean version.", "links": {} }, { "slug": "major-tom", "name": "Major TOM", "page": "https://rag-repo.org/source/major-tom", "url": "https://huggingface.co/Major-TOM", "category": "embedded-datasets", "categoryName": "Pre-embedded & RAG-ready", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "embedded", "format": [ "Parquet", "GeoParquet" ], "size": "100 B+ embedding vectors (plus tens of TB of source imagery)", "sizeTier": "huge", "licence": "CC-BY-SA-4.0", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "ESA Phi-lab", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "earth-observation", "satellite", "embeddings", "copernicus", "sentinel", "geospatial", "remote-sensing" ], "description": "An open, globally dense collection of embeddings and ML-ready imagery derived from Copernicus Sentinel data, published by ESA Phi-lab. The embedding expansions run to well over 100 billion vectors covering most of the Earth.", "links": { "documentation": "https://github.com/ESA-PhiLab/Major-TOM", "github": "https://github.com/ESA-PhiLab/Major-TOM", "huggingface": "https://huggingface.co/Major-TOM" } }, { "slug": "masakhane", "name": "Masakhane", "page": "https://rag-repo.org/source/masakhane", "url": "https://www.masakhane.io", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "small", "licence": "Varies (mostly CC BY / Apache 2.0)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Masakhane community", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "african-languages", "low-resource", "community", "benchmarks" ], "description": "A grassroots research community building natural language processing resources for African languages. It produces datasets, benchmarks, and models across dozens of African languages that are otherwise poorly served.", "links": { "github": "https://github.com/masakhane-io" } }, { "slug": "mason-oer-metafinder", "name": "Mason OER Metafinder", "page": "https://rag-repo.org/source/mason-oer-metafinder", "url": "https://oer.deepwebaccess.com/oer/desktop/en/search.html", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "web-portal" ], "ragReadiness": "raw", "format": [ "HTML" ], "size": null, "sizeTier": "unknown", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "George Mason University Libraries", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "education", "meta-resource", "search-tool", "open-educational-resources", "discovery" ], "description": "A discovery tool that runs a real-time simultaneous search across 21 different sources of open educational materials, rather than searching a static database. Useful for finding content to ingest, not a dataset in itself.", "links": {} }, { "slug": "mleb", "name": "Massive Legal Embedding Benchmark (MLEB)", "page": "https://rag-repo.org/source/mleb", "url": "https://isaacus.com/mleb", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "chunked", "format": [ "Parquet", "JSON" ], "size": "10 evaluation datasets", "sizeTier": "small", "licence": "Varies (permissive; components e.g. CC BY 4.0, code MIT)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Isaacus", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "legal", "embeddings", "retrieval", "benchmark", "evaluation", "question-answering" ], "description": "A benchmark for evaluating legal text embedding and retrieval models, built from ten expert-annotated datasets spanning six jurisdictions and three task types (search, zero-shot classification, and question answering).", "links": { "documentation": "https://huggingface.co/blog/isaacus/introducing-mleb", "github": "https://github.com/isaacus-dev/mleb", "huggingface": "https://huggingface.co/isaacus" } }, { "slug": "materials-project", "name": "Materials Project", "page": "https://rag-repo.org/source/materials-project", "url": "https://materialsproject.org", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "api" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "unknown", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "materials-science", "computed-properties", "api", "simulation", "research" ], "description": "A database of computed properties for known and predicted materials, produced through high-throughput density functional theory calculations (a physics method for modelling how electrons behave in a material). Free API access with registration.", "links": { "api": "https://api.materialsproject.org" } }, { "slug": "megamath", "name": "MegaMath", "page": "https://rag-repo.org/source/megamath", "url": "https://huggingface.co/datasets/LLM360/MegaMath", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "300 B+ tokens", "sizeTier": "medium", "licence": "ODC-By 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "LLM360", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "pretraining", "large-corpus", "web-crawl" ], "description": "An open mathematics pretraining dataset curated from diverse, maths-focused sources, with over 300 billion tokens. Among the largest open maths corpora available.", "links": {} }, { "slug": "merlot", "name": "MERLOT", "page": "https://rag-repo.org/source/merlot", "url": "https://www.merlot.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "web-portal" ], "ragReadiness": "raw", "format": [ "HTML" ], "size": null, "sizeTier": "unknown", "licence": "Varies by resource", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "MERLOT (California State University)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "education", "open-educational-resources", "catalogue", "teaching" ], "description": "A curated collection of online learning and support materials, plus content-creation tools, led by an international community of educators, learners, and researchers. Points to resources hosted elsewhere across many disciplines.", "links": {} }, { "slug": "ceda", "name": "Met Office DataPoint / CEDA", "page": "https://rag-repo.org/source/ceda", "url": "https://data.ceda.ac.uk", "category": "climate-earth", "categoryName": "Climate & earth observation", "accessType": "open", "accessPatterns": [ "bulk-download", "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Open Government Licence / varies", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Centre for Environmental Data Analysis", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "climate", "weather", "earth-observation", "uk", "satellite" ], "description": "The Centre for Environmental Data Analysis hosts UK atmospheric and Earth observation data, including Met Office archives, climate model outputs, and satellite data.", "links": {} }, { "slug": "mimic", "name": "MIMIC-III / MIMIC-IV", "page": "https://rag-repo.org/source/mimic", "url": "https://physionet.org/content/mimiciii/", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "limited", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [ "CSV" ], "size": "40,000+ ICU patients", "sizeTier": "small", "licence": "PhysioNet Credentialed Health Data License", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "MIT Laboratory for Computational Physiology", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "clinical", "critical-care", "ehr", "de-identified", "clinical-notes", "restricted" ], "description": "Freely accessible critical care databases with de-identified health data from more than 40,000 intensive care patients at Beth Israel Deaconess Medical Center. They include vital signs, lab results, medications, and clinical notes. Access needs credentialing and a data-use agreement.", "links": { "documentation": "https://physionet.org/content/mimiciv/" } }, { "slug": "miracl", "name": "MIRACL", "page": "https://rag-repo.org/source/miracl", "url": "https://project-miracl.github.io", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "chunked", "format": [], "size": "18 languages", "sizeTier": "small", "licence": "Apache 2.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "multilingual", "wikipedia", "evaluation" ], "description": "A multilingual retrieval benchmark built from human-annotated relevance judgements over Wikipedia articles, covering 18 languages. Widely used for evaluating retrieval within a single language across a broad set of languages.", "links": {} }, { "slug": "miriad", "name": "MIRIAD", "page": "https://rag-repo.org/source/miriad", "url": "https://huggingface.co/datasets/miriad/miriad-5.8M", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "chunked", "format": [ "Parquet" ], "size": "5.8M QA pairs", "sizeTier": "small", "licence": "ODC-By 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "ETH Medical AI Lab", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "biomedical", "medical-qa", "instruction-tuning", "retrieval", "hallucination-detection", "research" ], "description": "A million-scale medical instruction and retrieval dataset of roughly 5.8 million question-answer pairs, each grounded in a passage from peer-reviewed biomedical literature. Built for medical RAG, retrieval, and hallucination detection.", "links": { "documentation": "https://med-miriad.github.io/", "github": "https://github.com/eth-medical-ai-lab/MIRIAD", "huggingface": "https://huggingface.co/datasets/miriad/miriad-5.8M" } }, { "slug": "mit-opencourseware", "name": "MIT OpenCourseWare", "page": "https://rag-repo.org/source/mit-opencourseware", "url": "https://ocw.mit.edu", "category": "education", "categoryName": "Education & open learning", "accessType": "limited", "accessPatterns": [ "web-portal", "bulk-download" ], "ragReadiness": "raw", "format": [ "PDF", "HTML" ], "size": "2,500+ courses", "sizeTier": "unknown", "licence": "CC BY-NC-SA", "licenceUrl": null, "commercialUse": "prohibited", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Massachusetts Institute of Technology", "updateFrequency": null, "yearLaunched": 2001, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "education", "courseware", "lecture-notes", "cc-by-nc-sa", "higher-education" ], "description": "Teaching materials from more than 2,500 MIT undergraduate and graduate courses, published freely online. Includes lecture notes, assignments, and exams. Released under a NonCommercial licence, so it is free to use for non-commercial purposes only.", "links": {} }, { "slug": "mixturevitae", "name": "MixtureVitae", "page": "https://rag-repo.org/source/mixturevitae", "url": "https://huggingface.co/datasets/ontocord/MixtureVitae-v1", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "JSON", "Parquet" ], "size": "422 B tokens", "sizeTier": "large", "licence": "Varies (permissive-first, per-source)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Ontocord.AI", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "pretraining", "permissive", "public-domain", "government", "instruction-tuning", "large-corpus" ], "description": "A permissive-first, open pretraining corpus of roughly 422 billion tokens, drawn from public-domain, permissively licensed, and civic or government text, with sources organised into provenance-based risk tiers.", "links": { "documentation": "https://arxiv.org/abs/2509.25531", "github": "https://github.com/ontocord/mixturevitae", "huggingface": "https://huggingface.co/datasets/ontocord/MixtureVitae-v1" } }, { "slug": "common-voice", "name": "Mozilla Common Voice", "page": "https://rag-repo.org/source/common-voice", "url": "https://datacollective.mozillafoundation.org", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "web-portal", "bulk-download" ], "ragReadiness": "raw", "format": [ "MP3", "TSV" ], "size": "32,000+ hours across 131 languages", "sizeTier": "medium", "licence": "CC0 1.0", "licenceUrl": "https://creativecommons.org/publicdomain/zero/1.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Mozilla Foundation", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "v19.0", "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "speech", "multilingual", "crowdsourced", "cc0", "asr", "nonprofit" ], "description": "A crowdsourced speech platform from Mozilla that releases datasets under the CC0 licence. Volunteers read sentences aloud and other community members validate each recording. As of release 19.0 it holds 32,584 hours of speech across 131 languages, making it one of the largest openly licensed voice datasets available.", "links": {} }, { "slug": "ms-marco", "name": "MS MARCO", "page": "https://rag-repo.org/source/ms-marco", "url": "https://microsoft.github.io/msmarco/", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "limited", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "chunked", "format": [], "size": null, "sizeTier": "small", "licence": "MS MARCO non-commercial licence", "licenceUrl": null, "commercialUse": "prohibited", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Microsoft", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "question-answering", "passage-ranking", "non-commercial" ], "description": "One of the most widely used general-domain retrieval benchmarks, built from real Bing search queries with human-annotated passage relevance judgements. It carries a non-commercial licence: some model developers deliberately exclude MS MARCO from training because of its terms, so check carefully before any commercial use.", "links": {} }, { "slug": "mteb", "name": "MTEB (Massive Text Embedding Benchmark)", "page": "https://rag-repo.org/source/mteb", "url": "https://huggingface.co/spaces/mteb/leaderboard", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "chunked", "format": [], "size": null, "sizeTier": "unknown", "licence": "Apache 2.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "embeddings", "benchmark", "evaluation", "multilingual", "retrieval", "leaderboard" ], "description": "The Massive Text Embedding Benchmark, a common framework for comparing text embedding models (the numeric vectors that represent meaning for search and retrieval). It brings together SemEval, BEIR, and many other datasets across tasks such as retrieval, classification, and clustering. The MMTEB extension widens the scope to over 250 languages and more than 500 tasks.", "links": { "huggingface": "https://huggingface.co/spaces/mteb/leaderboard" } }, { "slug": "nasa-earthdata", "name": "NASA Earthdata", "page": "https://rag-repo.org/source/nasa-earthdata", "url": "https://www.earthdata.nasa.gov", "category": "climate-earth", "categoryName": "Climate & earth observation", "accessType": "open", "accessPatterns": [ "cloud-bucket", "api", "web-portal" ], "ragReadiness": "raw", "format": [], "size": "12,400+ datasets", "sizeTier": "huge", "licence": "Public domain (US Government)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "NASA", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "earth-observation", "satellite", "climate", "us", "api", "public-domain" ], "description": "The home for full and open access to NASA's Earth science data collections. The umbrella portal for the largest collection of freely available Earth science data in the world, with over 12,400 datasets spanning land, ocean, atmosphere, and cryosphere, collected by satellite missions, airborne campaigns, and ground stations.", "links": { "api": "https://cmr.earthdata.nasa.gov" } }, { "slug": "natural-earth", "name": "Natural Earth", "page": "https://rag-repo.org/source/natural-earth", "url": "https://www.naturalearthdata.com", "category": "geospatial", "categoryName": "Geospatial & mapping", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "cleaned", "format": [ "Shapefile", "GeoTIFF" ], "size": null, "sizeTier": "small", "licence": "Public domain", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Natural Earth (North American Cartographic Information Society)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "public-domain", "maps", "cartography", "vector", "raster", "boundaries" ], "description": "A public domain map dataset built for cartography, offered at three scales: 1:10m, 1:50m, and 1:110m. It bundles cultural data (borders, cities, roads), physical data (coastlines, rivers, lakes), and raster imagery, all ready to drop into a map.", "links": { "download": "https://www.naturalearthdata.com/downloads/" } }, { "slug": "natural-questions", "name": "Natural Questions", "page": "https://rag-repo.org/source/natural-questions", "url": "https://ai.google.com/research/NaturalQuestions", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "chunked", "format": [], "size": null, "sizeTier": "small", "licence": "CC BY-SA 3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Google", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "benchmark", "question-answering", "wikipedia", "google" ], "description": "A general-domain retrieval benchmark of real, anonymised Google search queries paired with Wikipedia passages that contain the answers. Alongside MS MARCO, one of the two most widely used retrieval benchmarks for general-domain evaluation.", "links": {} }, { "slug": "naturalproofs", "name": "NaturalProofs", "page": "https://rag-repo.org/source/naturalproofs", "url": "https://github.com/wellecks/naturalproofs", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": "32,000 theorems and proofs", "sizeTier": "tiny", "licence": "MIT (code); source licences vary", "licenceUrl": "https://github.com/wellecks/naturalproofs/blob/master/LICENSE", "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Sean Welleck", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "proofs", "theorems", "proofwiki", "formal-mathematics" ], "description": "A dataset of 32,000 theorem statements and proofs, 14,000 definitions, and 2,000 other pages including axioms and corollaries, drawn from ProofWiki, the Stacks Project, and mathematics textbooks.", "links": {} }, { "slug": "nell", "name": "NELL (Never-Ending Language Learner)", "page": "https://rag-repo.org/source/nell", "url": "http://rtw.ml.cmu.edu/rtw/", "category": "knowledge-graphs", "categoryName": "Knowledge graphs & structured data", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "small", "licence": "Research use", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Carnegie Mellon University", "updateFrequency": null, "yearLaunched": 2010, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "knowledge-graph", "machine-learning", "web-extraction", "research", "academic" ], "description": "A machine learning system from Carnegie Mellon that has been reading the web since 2010 and building a knowledge base as it goes. It extracts beliefs, entities, and the relationships between them from text, and keeps refining them over time.", "links": { "download": "http://rtw.ml.cmu.edu/rtw/resources" } }, { "slug": "nemotron-post-training-v2", "name": "Nemotron Post-Training Dataset v2", "page": "https://rag-repo.org/source/nemotron-post-training-v2", "url": "https://huggingface.co/datasets/nvidia/Nemotron-Post-Training-Dataset-v2", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": null, "sizeTier": "small", "licence": "CC-BY-4.0 (WildChat subset ODC-BY; StackOverflow subset CC-BY-SA)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "NVIDIA", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": "v2", "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "post-training", "fine-tuning", "synthetic-data", "multilingual", "reasoning", "instruction-following", "nvidia" ], "description": "NVIDIA's 2025 post-training set of prompts and synthetic responses for supervised fine-tuning and reinforcement learning, covering mathematics, code, STEM, reasoning, and instruction following, with the instruction data expanded into five additional languages.", "links": { "huggingface": "https://huggingface.co/datasets/nvidia/Nemotron-Post-Training-Dataset-v2" } }, { "slug": "nemotron-cc-v2", "name": "Nemotron-CC-v2", "page": "https://rag-repo.org/source/nemotron-cc-v2", "url": "https://huggingface.co/datasets/nvidia/Nemotron-CC-v2", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "limited", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "Parquet" ], "size": "~6.6 T tokens (10.3 TB)", "sizeTier": "large", "licence": "NVIDIA Data Agreement for Model Training", "licenceUrl": "https://huggingface.co/datasets/nvidia/Nemotron-CC-v2/blob/main/LICENSE.md", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "NVIDIA", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": "v2", "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "pretraining", "common-crawl", "synthetic-data", "multilingual", "english", "nvidia" ], "description": "NVIDIA's rebuilt Common Crawl corpus for LLM pretraining, adding eight fresh 2024 to 2025 snapshots, synthetic rephrasing, and multilingual synthetic question and answer data, with mathematics and code content preserved.", "links": { "huggingface": "https://huggingface.co/datasets/nvidia/Nemotron-CC-v2" } }, { "slug": "ngss", "name": "Next Generation Science Standards", "page": "https://rag-repo.org/source/ngss", "url": "https://www.nextgenscience.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "bulk-download", "web-crawl" ], "ragReadiness": "cleaned", "format": [ "PDF", "HTML" ], "size": null, "sizeTier": "tiny", "licence": "Free to use with attribution; commercial use permitted subject to WestEd review and trademark terms (not Creative Commons)", "licenceUrl": "https://www.nextgenscience.org/ngss-trademarks-and-copyright/ngss-trademarks-and-copyright", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "WestEd", "updateFrequency": null, "yearLaunched": 2013, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "science-standards", "k-12", "curriculum-framework", "education", "us", "performance-expectations" ], "description": "US K-12 science standards developed by a consortium of states and released in 2013. A curriculum framework of performance expectations arranged by grade band and disciplinary core idea, free to use with attribution and browsable online or downloadable as PDFs.", "links": {} }, { "slug": "nhs-england-digital", "name": "NHS England Digital", "page": "https://rag-repo.org/source/nhs-england-digital", "url": "https://digital.nhs.uk/data", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "bulk-download", "web-portal" ], "ragReadiness": "cleaned", "format": [ "CSV", "Excel" ], "size": null, "sizeTier": "unknown", "licence": "Open Government Licence v3.0", "licenceUrl": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "NHS England", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "health", "social-care", "statistics", "open-government-licence" ], "description": "The national provider of information and data on health and social care in England, covering general practices, hospital-level mortality, mental health, population health, and NHS outcomes. Published under the Open Government Licence.", "links": {} }, { "slug": "nlp-datasets", "name": "NLP Datasets", "page": "https://rag-repo.org/source/nlp-datasets", "url": "https://github.com/niderhoff/nlp-datasets", "category": "curated-lists", "categoryName": "Curated lists & meta-resources", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "raw", "format": [ "Markdown" ], "size": null, "sizeTier": "tiny", "licence": "Check source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "niderhoff (community)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "awesome-list", "nlp", "text-corpora", "directory", "github" ], "description": "An alphabetical list of free and public domain text datasets for natural language processing, covering corpora, dialogue, sentiment, and summarisation. Handy when you want text-heavy data to build or evaluate a RAG system.", "links": { "github": "https://github.com/niderhoff/nlp-datasets" } }, { "slug": "noaa-nodd", "name": "NOAA Open Data Dissemination (NODD)", "page": "https://rag-repo.org/source/noaa-nodd", "url": "https://www.noaa.gov/nodd", "category": "climate-earth", "categoryName": "Climate & earth observation", "accessType": "open", "accessPatterns": [ "cloud-bucket", "api" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "huge", "licence": "Public domain (US Government)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "NOAA", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "climate", "weather", "oceanic", "us", "bulk-download", "public-domain" ], "description": "Free, full, and open access to NOAA data through partner platforms including Amazon Web Services, Google, and Microsoft Azure. Covers climate data records and atmospheric, oceanic, and terrestrial datasets from one of the largest climate archives in the world.", "links": {} }, { "slug": "nomis", "name": "Nomis", "page": "https://rag-repo.org/source/nomis", "url": "https://www.nomisweb.co.uk", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "api", "web-portal" ], "ragReadiness": "cleaned", "format": [ "CSV", "JSON" ], "size": null, "sizeTier": "unknown", "licence": "Open Government Licence v3.0", "licenceUrl": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Office for National Statistics", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "census", "labour-market", "statistics", "open-government-licence", "api" ], "description": "A service from the Office for National Statistics holding detailed data on the national and local labour market, covering official census and labour market statistics. Offers an API for programmatic access.", "links": { "api": "https://www.nomisweb.co.uk/api/v01/help" } }, { "slug": "obelics", "name": "OBELICS", "page": "https://rag-repo.org/source/obelics", "url": "https://huggingface.co/datasets/HuggingFaceM4/OBELICS", "category": "multimodal", "categoryName": "Multimodal & image-text", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [], "size": "141M documents", "sizeTier": "medium", "licence": "CC BY 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Hugging Face", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "image-text", "interleaved", "web-scale", "multimodal", "english" ], "description": "A web-scale dataset of 141M multimodal English web documents that interleave text and images, containing 353M images and 115 billion tokens. Unlike image-caption pair datasets, it keeps whole documents with text and images in their original order. Used to train the IDEFICS models.", "links": {} }, { "slug": "oecd-data", "name": "OECD Data", "page": "https://rag-repo.org/source/oecd-data", "url": "https://data.oecd.org", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "api", "web-portal" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "unknown", "licence": "OECD terms (generally permissive with attribution)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "OECD", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "statistics", "economics", "development", "global", "api", "policy" ], "description": "Reports, data, and publications on the economic, environmental, and social conditions in hundreds of countries. Includes International Development Statistics covering aid volume, origin, and types, plus the Creditor Reporting System with detailed information on individual aid activities.", "links": {} }, { "slug": "oer-commons", "name": "OER Commons", "page": "https://rag-repo.org/source/oer-commons", "url": "https://www.oercommons.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "web-portal", "web-crawl" ], "ragReadiness": "mixed", "format": [ "HTML" ], "size": "500,000+ resources", "sizeTier": "unknown", "licence": "Varies by resource (CC licences)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Institute for the Study of Knowledge Management in Education (ISKME)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "education", "open-educational-resources", "cc-licensed", "library", "teaching" ], "description": "A digital library of over 500,000 open educational resources gathered from institutions around the world. Aggregates full courses, modules, lesson plans, videos, simulations, and assessments across every discipline and education level.", "links": {} }, { "slug": "omnilingual-asr-corpus", "name": "Omnilingual ASR Corpus", "page": "https://rag-repo.org/source/omnilingual-asr-corpus", "url": "https://huggingface.co/datasets/facebook/omnilingual-asr-corpus", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~3,350 hours across ~348 languages", "sizeTier": "medium", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "AI at Meta (FAIR)", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "speech", "transcription", "multilingual", "low-resource", "audio", "meta" ], "description": "A Meta FAIR corpus of transcribed spontaneous speech covering roughly 348 under-served, low-resource languages, released in 2025 under CC BY 4.0. Around 3,350 hours of audio paired with human transcriptions.", "links": { "documentation": "https://ai.meta.com/research/publications/omnilingual-asr-advancing-automatic-speech-recognition-for-1600-languages/", "huggingface": "https://huggingface.co/datasets/facebook/omnilingual-asr-corpus" } }, { "slug": "ons", "name": "ONS (Office for National Statistics)", "page": "https://rag-repo.org/source/ons", "url": "https://www.ons.gov.uk", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "CSV", "JSON", "XLSX" ], "size": null, "sizeTier": "unknown", "licence": "Open Government Licence v3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Office for National Statistics", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "government", "statistics", "census", "economics" ], "description": "The UK's official statistics body. It publishes Census data, economic indicators, population estimates, labour market figures, and more, making it the authoritative source for UK statistics in a RAG system.", "links": {} }, { "slug": "open-beauty-facts", "name": "Open Beauty Facts / Open Products Facts", "page": "https://rag-repo.org/source/open-beauty-facts", "url": "https://world.openbeautyfacts.org", "category": "consumer-product", "categoryName": "Consumer & product data", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "tiny", "licence": "ODbL", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Open Food Facts", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "cosmetics", "beauty", "products", "barcode", "crowdsourced", "open-data" ], "description": "Sister projects to Open Food Facts that apply the same collaborative model to cosmetics (Open Beauty Facts) and general consumer goods (Open Products Facts). Contributors scan barcodes and photograph packaging to build an open database of ingredients and product information.", "links": {} }, { "slug": "open-food-facts", "name": "Open Food Facts", "page": "https://rag-repo.org/source/open-food-facts", "url": "https://world.openfoodfacts.org", "category": "consumer-product", "categoryName": "Consumer & product data", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "JSON", "CSV" ], "size": "Millions of products across 140+ countries", "sizeTier": "small", "licence": "ODbL (database), Database Contents Licence (contents), CC BY-SA (images)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Open Food Facts", "updateFrequency": "Nightly", "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "food", "nutrition", "barcode", "crowdsourced", "open-data", "products" ], "description": "A collaborative, free, and open database of ingredients, nutrition facts, and information on food products from around the world. Contributors scan barcodes and photograph ingredient lists and nutrition tables, following the model of Wikipedia and OpenStreetMap. Covers millions of products across 140+ countries.", "links": {} }, { "slug": "open-library", "name": "Open Library", "page": "https://rag-repo.org/source/open-library", "url": "https://openlibrary.org", "category": "books-literature", "categoryName": "Books & literature", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "JSON", "CSV" ], "size": "20M+ catalogue records", "sizeTier": "small", "licence": "CC0 (metadata); lending terms for full text", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Internet Archive", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "books", "metadata", "catalogue", "nonprofit", "api" ], "description": "An Internet Archive project building a web page for every book ever published. It holds more than 20 million catalogue records and lends many titles digitally, making it a rich source of book metadata for RAG systems.", "links": { "api": "https://openlibrary.org/developers/api" } }, { "slug": "omat24", "name": "Open Materials 2024 (OMat24)", "page": "https://rag-repo.org/source/omat24", "url": "https://huggingface.co/datasets/fairchem/OMAT24", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "raw", "format": [ "LMDB", "ASE DB" ], "size": "110M+ DFT calculations", "sizeTier": "medium", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Meta FAIR Chemistry", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": "2024", "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "materials-science", "chemistry", "dft", "inorganic-materials", "machine-learning-potentials", "meta-fair" ], "description": "Meta FAIR's open dataset of more than 110 million DFT calculations of inorganic materials, sampled for structural and compositional diversity. Models trained on it top the MatBench-Discovery leaderboard for stability and formation-energy prediction.", "links": { "github": "https://github.com/facebookresearch/fairchem", "huggingface": "https://huggingface.co/datasets/fairchem/OMAT24" } }, { "slug": "omol25", "name": "Open Molecules 2025 (OMol25)", "page": "https://rag-repo.org/source/omol25", "url": "https://huggingface.co/facebook/OMol25", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "limited", "accessPatterns": [ "huggingface" ], "ragReadiness": "raw", "format": [ "LMDB", "ASE DB" ], "size": "100M+ DFT calculations", "sizeTier": "medium", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Meta FAIR Chemistry", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": "2025", "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "chemistry", "molecular-dynamics", "dft", "machine-learning-potentials", "meta-fair", "scientific" ], "description": "A large-scale dataset of more than 100 million high-accuracy DFT calculations spanning 83 elements, built by Meta FAIR to train machine-learning interatomic potentials for molecular systems.", "links": { "documentation": "https://fair-chem.github.io/omol25/", "github": "https://github.com/facebookresearch/fairchem", "huggingface": "https://huggingface.co/facebook/OMol25" } }, { "slug": "open-ragbench", "name": "Open RAGBench (Vectara)", "page": "https://rag-repo.org/source/open-ragbench", "url": "https://huggingface.co/datasets/vectara/open_ragbench", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "Parquet" ], "size": "1,000 papers, 3,045 QA pairs", "sizeTier": "tiny", "licence": "Apache 2.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Vectara", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "evaluation", "benchmark", "arxiv", "multimodal", "scientific", "question-answering" ], "description": "A RAG evaluation benchmark from Vectara built on 1,000 arXiv papers, with multimodal content extraction and 3,045 question-and-answer pairs across scientific domains. Designed to test retrieval and answer quality on real research documents rather than short, simplified passages.", "links": { "huggingface": "https://huggingface.co/datasets/vectara/open_ragbench" } }, { "slug": "open-reaction-database", "name": "Open Reaction Database", "page": "https://rag-repo.org/source/open-reaction-database", "url": "https://open-reaction-database.org", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "git-repo", "web-portal" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "small", "licence": "CC BY-SA 4.0", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "chemistry", "reactions", "machine-learning", "synthesis", "share-alike" ], "description": "An open-access chemical reaction database built to support machine learning for reaction prediction, synthesis planning, and experiment design. A collaborative effort spanning pharmaceutical companies, academia, and technology firms.", "links": {} }, { "slug": "open-targets", "name": "Open Targets Platform", "page": "https://rag-repo.org/source/open-targets", "url": "https://platform.opentargets.org", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "open", "accessPatterns": [ "api", "cloud-bucket", "bulk-download" ], "ragReadiness": "mixed", "format": [ "Parquet", "JSON" ], "size": "~865,000 target-disease pairs", "sizeTier": "small", "licence": "CC0 1.0", "licenceUrl": "https://creativecommons.org/publicdomain/zero/1.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Open Targets (EMBL-EBI, Wellcome Sanger Institute, GSK and partners)", "updateFrequency": null, "yearLaunched": 2016, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "biomedical", "drug-discovery", "genomics", "knowledge-base", "api", "bulk-download" ], "description": "An open knowledge base of scored target-disease associations for drug discovery, integrating genetics, genomics, literature, and drug evidence from many public sources. Accessible via REST and GraphQL APIs, BigQuery, and bulk Parquet downloads.", "links": { "documentation": "https://platform-docs.opentargets.org/", "api": "https://api.platform.opentargets.org/api/v4/graphql", "download": "https://platform.opentargets.org/downloads", "github": "https://github.com/opentargets" } }, { "slug": "open-textbook-library", "name": "Open Textbook Library", "page": "https://rag-repo.org/source/open-textbook-library", "url": "https://open.umn.edu/opentextbooks", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "web-portal" ], "ragReadiness": "raw", "format": [ "PDF", "HTML" ], "size": null, "sizeTier": "small", "licence": "Varies (CC licences)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "University of Minnesota", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "education", "open-textbooks", "higher-education", "cc-licensed", "catalogue" ], "description": "A searchable catalogue of free, openly licensed textbooks, developed by the University of Minnesota. Open textbooks here are funded, published, and licensed to be freely used, adapted, and distributed, with faculty reviews on each entry.", "links": {} }, { "slug": "openalex", "name": "OpenAlex", "page": "https://rag-repo.org/source/openalex", "url": "https://openalex.org", "category": "academic", "categoryName": "Academic & scientific literature", "accessType": "open", "accessPatterns": [ "api", "cloud-bucket" ], "ragReadiness": "raw", "format": [ "JSON" ], "size": "Hundreds of millions of works", "sizeTier": "medium", "licence": "CC0 (metadata)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "OurResearch", "updateFrequency": "Monthly", "yearLaunched": 2022, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "academic", "metadata", "citations", "open-data", "research", "api" ], "description": "A free, open catalogue of the world's scholarly works, authors, venues, institutions, and research topics. It succeeds Microsoft Academic Graph and holds hundreds of millions of works with rich metadata and citation links.", "links": {} }, { "slug": "opencorporates", "name": "OpenCorporates", "page": "https://rag-repo.org/source/opencorporates", "url": "https://opencorporates.com", "category": "consumer-product", "categoryName": "Consumer & product data", "accessType": "limited", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "medium", "licence": "Mixed (some open data, commercial licensing for bulk and API access)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "OpenCorporates", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "companies", "corporate", "registry", "directors", "regulatory", "open-data" ], "description": "A global database of companies, their directors, and regulatory filings. The largest open database of companies in the world, though bulk and API access is commercially licensed rather than freely reusable.", "links": {} }, { "slug": "opencyc", "name": "OpenCyc", "page": "https://rag-repo.org/source/opencyc", "url": "https://cyc.com", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "mixed", "format": [], "size": null, "sizeTier": "tiny", "licence": "Apache 2.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Cycorp (archived)", "updateFrequency": "Archived", "yearLaunched": 1984, "isArchived": true, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "knowledge-base", "ontology", "common-sense", "archived", "reasoning" ], "description": "The open release of Cyc, one of the oldest attempts to hand-build common-sense knowledge for machines. It holds hundreds of thousands of concepts and millions of assertions about how the everyday world fits together. Now archived, but the data is still available.", "links": {} }, { "slug": "openlearn", "name": "OpenLearn (Open University)", "page": "https://rag-repo.org/source/openlearn", "url": "https://www.open.edu/openlearn/", "category": "education", "categoryName": "Education & open learning", "accessType": "limited", "accessPatterns": [ "web-crawl", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "HTML" ], "size": null, "sizeTier": "tiny", "licence": "CC BY-NC-SA 4.0", "licenceUrl": null, "commercialUse": "prohibited", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "The Open University", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "education", "uk", "cc-by-nc-sa", "open-university", "courses" ], "description": "The Open University's free learning platform, offering courses and educational resources across a wide range of subjects and academic levels. A strong UK-based option for education content. Released under a NonCommercial licence, so commercial use is not permitted.", "links": {} }, { "slug": "opensanctions", "name": "OpenSanctions", "page": "https://rag-repo.org/source/opensanctions", "url": "https://www.opensanctions.org", "category": "consumer-product", "categoryName": "Consumer & product data", "accessType": "limited", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "small", "licence": "CC BY-NC 4.0 (free tier); commercial licence available", "licenceUrl": null, "commercialUse": "prohibited", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "OpenSanctions", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "sanctions", "compliance", "risk", "knowledge-graph", "open-data" ], "description": "A database of sanctioned entities, politically exposed persons, and related risk data, aggregated from official sources worldwide. Links entities to OpenCorporates where both hold the same entity, so you can combine sanctions data with company control information.", "links": {} }, { "slug": "openslr", "name": "OpenSLR", "page": "https://rag-repo.org/source/openslr", "url": "https://www.openslr.org", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "bulk-download", "web-portal" ], "ragReadiness": "mixed", "format": [ "WAV", "FLAC" ], "size": null, "sizeTier": "unknown", "licence": "Varies by dataset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "speech", "multilingual", "asr", "hosting", "corpora" ], "description": "Open Speech and Language Resources, a hosting site for speech and language datasets, software, and models. It is the home of LibriSpeech and dozens of other language-specific speech corpora, making it a central catalogue for finding openly available voice data.", "links": {} }, { "slug": "openstates", "name": "OpenStates", "page": "https://rag-repo.org/source/openstates", "url": "https://openstates.org", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "small", "licence": "CC0 (data); MIT (code)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": "Continuous", "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "us", "legislation", "open-source", "api", "bulk-download", "public-domain" ], "description": "An open-source platform tracking US state legislation in real time. Bulk data and API access for bills, legislators, and votes across all 50 states.", "links": {} }, { "slug": "openstax", "name": "OpenStax", "page": "https://rag-repo.org/source/openstax", "url": "https://openstax.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "cleaned", "format": [ "PDF", "HTML" ], "size": null, "sizeTier": "small", "licence": "Primarily CC BY, some titles CC BY-NC-SA", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "OpenStax (Rice University)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "textbooks", "education", "open-textbooks", "cc-by", "higher-education", "nonprofit" ], "description": "Openly licensed college textbooks from a non-profit initiative of Rice University. Peer-reviewed titles cover common undergraduate subjects, free to read online and available in low-cost print, and are used by millions of students each month.", "links": {} }, { "slug": "openstreetmap", "name": "OpenStreetMap", "page": "https://rag-repo.org/source/openstreetmap", "url": "https://www.openstreetmap.org", "category": "geospatial", "categoryName": "Geospatial & mapping", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "PBF", "XML" ], "size": "Full planet file is roughly 140 GB compressed", "sizeTier": "medium", "licence": "ODbL 1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "OpenStreetMap Foundation", "updateFrequency": "Continuous", "yearLaunched": 2004, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "crowdsourced", "maps", "geographic", "open-data", "nonprofit", "points-of-interest" ], "description": "A crowdsourced map of the world, built and maintained by a community of volunteers. It holds detailed geographic data on roads, buildings, land use, points of interest, and much more, which you can download as a full planet file or as smaller regional extracts.", "links": { "api": "https://overpass-api.de", "download": "https://planet.openstreetmap.org" } }, { "slug": "opensubtitles", "name": "OpenSubtitles", "page": "https://rag-repo.org/source/opensubtitles", "url": "https://opus.nlpl.eu/OpenSubtitles/", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "limited", "accessPatterns": [ "bulk-download", "huggingface" ], "ragReadiness": "cleaned", "format": [ "TXT", "XML" ], "size": "2.6 billion sentences across 60 languages", "sizeTier": "medium", "licence": "Varies (subtitle content); compilation free for research", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "OPUS", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "subtitles", "parallel-corpus", "conversational", "translation" ], "description": "A large corpus of translated movie and TV subtitles. The latest version covers 60 languages with 2.6 billion sentences in total. It is valuable for conversational and colloquial language, and for parallel translation data.", "links": {} }, { "slug": "openthoughts3", "name": "OpenThoughts3-1.2M", "page": "https://rag-repo.org/source/openthoughts3", "url": "https://huggingface.co/datasets/open-thoughts/OpenThoughts3-1.2M", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~60 GB (1.2M rows)", "sizeTier": "medium", "licence": "Apache 2.0", "licenceUrl": "https://www.apache.org/licenses/LICENSE-2.0", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Open Thoughts", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "reasoning", "chain-of-thought", "mathematics", "synthetic", "fine-tuning", "code", "science" ], "description": "An open reasoning dataset of 1.2 million rows (roughly 850k maths, 250k code and 100k science questions) each paired with a chain-of-thought answer. Released by the Open Thoughts team in June 2025, it is the training set behind OpenThinker3 and is licensed Apache 2.0.", "links": { "documentation": "https://arxiv.org/abs/2506.04178", "github": "https://github.com/open-thoughts/open-thoughts" } }, { "slug": "openwebmath", "name": "OpenWebMath", "page": "https://rag-repo.org/source/openwebmath", "url": "https://huggingface.co/datasets/open-web-math/open-web-math", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "14.7 B tokens across 6.3 M documents", "sizeTier": "medium", "licence": "ODC-By 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "web-crawl", "common-crawl", "pretraining", "physics", "english" ], "description": "A corpus of high-quality mathematical text extracted from the web. Filtered from over 200 billion HTML pages on Common Crawl down to 6.3 million documents totalling 14.7 billion tokens. Mostly mathematics, with substantial coverage of physics, computer science, and statistics.", "links": {} }, { "slug": "ordnance-survey-opendata", "name": "Ordnance Survey OpenData", "page": "https://rag-repo.org/source/ordnance-survey-opendata", "url": "https://www.ordnancesurvey.co.uk/products/open-data", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "web-portal", "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [ "GeoPackage", "GML", "Shapefile", "CSV" ], "size": null, "sizeTier": "small", "licence": "Open Government Licence v3.0", "licenceUrl": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Ordnance Survey", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "geospatial", "mapping", "boundaries", "postcodes", "open-government-licence" ], "description": "A portfolio of free geospatial datasets covering Great Britain from the national mapping agency, available for commercial and personal reuse under the Open Government Licence. Provides mapping, boundary, and location data through the OS Data Hub in formats including GeoPackage, GML, Shapefile, and CSV.", "links": {} }, { "slug": "oscar", "name": "OSCAR", "page": "https://rag-repo.org/source/oscar", "url": "https://oscar-project.org", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "raw", "format": [ "JSONL" ], "size": null, "sizeTier": "large", "licence": "CC0 (metadata); content subject to original terms", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "OSCAR Project", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "multilingual", "pretraining", "language-classification" ], "description": "A multilingual web corpus (its name stands for Open Super-large Crawled Aggregated coRpus) extracted from Common Crawl with per-document language classification. Covers more than 150 languages, useful for non-English and cross-lingual RAG.", "links": {} }, { "slug": "our-world-in-data", "name": "Our World in Data", "page": "https://rag-repo.org/source/our-world-in-data", "url": "https://ourworldindata.org", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "api", "bulk-download", "web-portal" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "tiny", "licence": "CC BY 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Our World in Data", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "statistics", "data-visualisation", "global", "research", "api", "csv" ], "description": "Research and interactive visualisations on global problems, built by aggregating and harmonising data from official sources. Every chart can be downloaded as a ZIP containing a CSV file, JSON metadata, and a README, and the underlying data is available programmatically via an API. Full provenance is documented for every dataset.", "links": {} }, { "slug": "overture-maps", "name": "Overture Maps Foundation", "page": "https://rag-repo.org/source/overture-maps", "url": "https://overturemaps.org", "category": "geospatial", "categoryName": "Geospatial & mapping", "accessType": "open", "accessPatterns": [ "cloud-bucket" ], "ragReadiness": "cleaned", "format": [ "GeoParquet" ], "size": null, "sizeTier": "medium", "licence": "ODbL 1.0 and CDLA Permissive 2.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Overture Maps Foundation (Linux Foundation)", "updateFrequency": "Monthly", "yearLaunched": 2022, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "maps", "open-data", "interoperable", "places", "buildings", "collaboration" ], "description": "An open map data project run under the Linux Foundation by a group of industry members. It combines and harmonises data from OpenStreetMap, Microsoft, Meta, and others into consistent, interoperable map layers you can build on freely.", "links": { "documentation": "https://docs.overturemaps.org" } }, { "slug": "patentsview", "name": "PatentsView", "page": "https://rag-repo.org/source/patentsview", "url": "https://patentsview.org", "category": "patents", "categoryName": "Patents & intellectual property", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "CSV", "JSON" ], "size": null, "sizeTier": "small", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "USPTO", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "patents", "us", "uspto", "api", "bulk-download", "structured" ], "description": "A platform from the US Patent and Trademark Office (USPTO) for exploring US patent data, with visualisation and analysis tools. It connects patents, inventors, organisations, and locations in a regularly updated database, and offers both an API and complete bulk data tables.", "links": { "api": "https://patentsview.org", "download": "https://patentsview.org" } }, { "slug": "pd12m", "name": "PD12M (Public Domain 12M)", "page": "https://rag-repo.org/source/pd12m", "url": "https://huggingface.co/datasets/Spawning/PD12M", "category": "multimodal", "categoryName": "Multimodal & image-text", "accessType": "open", "accessPatterns": [ "huggingface", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": "12.4M image-caption pairs", "sizeTier": "large", "licence": "CDLA-Permissive-2.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Spawning", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "image-text", "public-domain", "cc0", "multimodal", "commercial-friendly" ], "description": "An image-text dataset built only from materials marked with a Public Domain Mark or released under Creative Commons Zero (CC0). 12.4M image-caption pairs, with a 3.3M subset called PD3M, sized to match the Conceptual Captions datasets while staying copyright-clean for commercial use.", "links": {} }, { "slug": "physionet", "name": "PhysioNet", "page": "https://rag-repo.org/source/physionet", "url": "https://physionet.org", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "open", "accessPatterns": [ "web-portal", "bulk-download" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Varies by dataset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "MIT Laboratory for Computational Physiology", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "clinical", "physiological-signals", "ecg", "eeg", "medical-research", "repository" ], "description": "A repository of freely available medical research data, including physiological signals such as ECG and EEG, clinical databases, and related software. It hosts MIMIC and many other biomedical datasets, so it is a central hub for clinical RAG source material.", "links": {} }, { "slug": "pile-of-law", "name": "Pile of Law", "page": "https://rag-repo.org/source/pile-of-law", "url": "https://huggingface.co/datasets/pile-of-law/pile-of-law", "category": "legal", "categoryName": "Legal & regulatory", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSONL" ], "size": "~256 GB", "sizeTier": "medium", "licence": "Varies by subset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Pile of Law authors", "updateFrequency": null, "yearLaunched": 2022, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "legal", "english", "us", "pretraining", "research" ], "description": "A 256 GB dataset of open English-language legal and administrative text, covering court opinions, contracts, administrative rules, and legislative records. A ready starting point for building a legal RAG system without assembling sources yourself.", "links": { "huggingface": "https://huggingface.co/datasets/pile-of-law/pile-of-law" } }, { "slug": "pleias-rag-resources", "name": "PleIAs RAG-Resources", "page": "https://rag-repo.org/source/pleias-rag-resources", "url": "https://huggingface.co/datasets/PleIAs/RAG-Resources", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [], "size": null, "sizeTier": "unknown", "licence": "Varies", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "PleIAs", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "evaluation", "collection", "curated", "pleias", "question-answering" ], "description": "A curated collection of datasets and resources assembled by PleIAs for building and evaluating RAG applications. A useful starting point when you want ready-made material to test retrieval pipelines without hunting for datasets one by one.", "links": { "huggingface": "https://huggingface.co/datasets/PleIAs/RAG-Resources" } }, { "slug": "project-gutenberg", "name": "Project Gutenberg", "page": "https://rag-repo.org/source/project-gutenberg", "url": "https://www.gutenberg.org", "category": "books-literature", "categoryName": "Books & literature", "accessType": "open", "accessPatterns": [ "bulk-download", "web-portal" ], "ragReadiness": "raw", "format": [ "TXT", "EPUB", "HTML" ], "size": "70,000+ ebooks (roughly 55,000 in English)", "sizeTier": "small", "licence": "Public domain (mostly); check per-work copyright status", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Project Gutenberg Literary Archive Foundation", "updateFrequency": null, "yearLaunched": 1971, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "public-domain", "books", "literature", "english", "multilingual", "nonprofit" ], "description": "A volunteer effort to digitise and archive public domain books, with more than 70,000 free ebooks. Mostly English, but it covers many languages. One of the oldest digital library projects, so it is a clean, permissively licensed source of full-text literature for RAG.", "links": {} }, { "slug": "proof-pile-2", "name": "Proof-Pile-2", "page": "https://rag-repo.org/source/proof-pile-2", "url": "https://huggingface.co/datasets/EleutherAI/proof-pile-2", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSONL" ], "size": "55 B tokens", "sizeTier": "medium", "licence": "Varies by subset", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "EleutherAI", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "2", "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "arxiv", "formal-mathematics", "pretraining", "scientific" ], "description": "A 55 billion token dataset of mathematical and scientific documents drawn from arXiv, OpenWebMath, and AlgebraicStack. The AlgebraicStack subset alone contributes 11 billion tokens covering numerical computing, computer algebra, and formal mathematics.", "links": {} }, { "slug": "pubchem", "name": "PubChem", "page": "https://rag-repo.org/source/pubchem", "url": "https://pubchem.ncbi.nlm.nih.gov", "category": "chemistry-materials", "categoryName": "Chemistry, materials & life sciences", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "SDF", "CSV", "RDF" ], "size": "300M+ substances, 100M+ compounds, 300M+ bioactivities", "sizeTier": "large", "licence": "Public domain (US Government)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "US National Institutes of Health", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "chemistry", "drug-discovery", "us-government", "public-domain", "structured", "bioactivity" ], "description": "An open chemistry database run by the US National Institutes of Health. Holds over 300M substances, 100M compounds, and almost 300M recorded bioactivities, covering chemical structures, identifiers, physical properties, patents, and health, safety, and toxicity data. Available in SDF, CSV, and other formats.", "links": {} }, { "slug": "pubmed", "name": "PubMed / PubMed Central (PMC)", "page": "https://rag-repo.org/source/pubmed", "url": "https://pubmed.ncbi.nlm.nih.gov", "category": "academic", "categoryName": "Academic & scientific literature", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "XML", "TXT" ], "size": "36M+ citations", "sizeTier": "medium", "licence": "Public domain (metadata); individual article licences vary", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "US National Library of Medicine", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "biomedical", "health", "full-text", "us", "research", "bulk-download" ], "description": "The US National Library of Medicine's database of biomedical and life sciences literature. PubMed indexes over 36 million citations and abstracts, and PubMed Central (PMC) adds free full-text access to a growing subset. You can download the data in bulk over FTP.", "links": { "documentation": "https://www.ncbi.nlm.nih.gov/pmc/" } }, { "slug": "rag-mini-wikipedia", "name": "RAG-Mini-Wikipedia", "page": "https://rag-repo.org/source/rag-mini-wikipedia", "url": "https://huggingface.co/datasets/rag-datasets/rag-mini-wikipedia", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "chunked", "format": [ "Parquet" ], "size": "918 QA pairs, 3,200 passages", "sizeTier": "tiny", "licence": "CC BY 3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "evaluation", "question-answering", "wikipedia", "benchmark", "small" ], "description": "A small evaluation set that pairs 918 question-and-answer pairs with a matching corpus of 3,200 Wikipedia passages. Built specifically for testing RAG systems, it is small enough to run quick evaluation loops while still covering realistic retrieval questions.", "links": { "huggingface": "https://huggingface.co/datasets/rag-datasets/rag-mini-wikipedia" } }, { "slug": "ragtruth", "name": "RAGTruth", "page": "https://rag-repo.org/source/ragtruth", "url": "https://github.com/ParticleMedia/RAGTruth", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "git-repo" ], "ragReadiness": "cleaned", "format": [ "JSONL" ], "size": "~18,000 annotated responses (~14,300 hallucination spans)", "sizeTier": "tiny", "licence": "MIT", "licenceUrl": "https://github.com/ParticleMedia/RAGTruth/blob/main/LICENSE", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "ParticleMedia", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "hallucination-detection", "faithfulness", "benchmark", "annotated", "question-answering", "summarisation" ], "description": "A word-level hallucination corpus of roughly 18,000 LLM responses generated in a RAG setting, each manually annotated for hallucinated spans across question answering, summarisation, and data-to-text tasks. A reference corpus for training and evaluating faithfulness detectors.", "links": { "github": "https://github.com/ParticleMedia/RAGTruth" } }, { "slug": "rcsb-protein-data-bank", "name": "RCSB Protein Data Bank", "page": "https://rag-repo.org/source/rcsb-protein-data-bank", "url": "https://www.rcsb.org", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "mmCIF", "PDB", "PDBML", "JSON" ], "size": "250,000+ experimental structures plus 1M+ computed models", "sizeTier": "medium", "licence": "CC0 1.0 Universal (public domain dedication)", "licenceUrl": "https://www.rcsb.org/pages/policies", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "RCSB PDB (wwPDB partnership)", "updateFrequency": "Weekly", "yearLaunched": 1971, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "proteins", "structural-biology", "bioinformatics", "structured", "api", "bulk-download", "public-domain" ], "description": "The canonical open archive of experimentally determined 3D biomolecular structures, over 250,000 of them, served alongside more than a million computed structure models. It is the ground truth of structural biology, released into the public domain under CC0.", "links": { "documentation": "https://www.rcsb.org/docs", "api": "https://data.rcsb.org", "download": "https://www.rcsb.org/downloads" } }, { "slug": "redpajama", "name": "RedPajama", "page": "https://rag-repo.org/source/redpajama", "url": "https://github.com/togethercomputer/RedPajama-Data", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "mixed", "format": [ "JSONL", "Parquet" ], "size": "V1: 1.2T tokens; V2: 100+T tokens", "sizeTier": "huge", "licence": "Apache 2.0 (code); mixed (data sources)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Together", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "pretraining", "multi-source", "open-reproduction", "quality-signals" ], "description": "An open reproduction of the LLaMA training data from Together. V1 aggregates Wikipedia, books, arXiv, GitHub, Common Crawl, and Stack Exchange; V2 is a web-only corpus of more than 100T tokens with 46 quality signals per document.", "links": { "github": "https://github.com/togethercomputer/RedPajama-Data" } }, { "slug": "refinecode", "name": "RefineCode (OpenCoder)", "page": "https://rag-repo.org/source/refinecode", "url": "https://huggingface.co/datasets/OpenCoder-LLM/RefineCode-code-corpus-meta", "category": "code", "categoryName": "Code & technical documentation", "accessType": "open", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~960 B tokens (Hugging Face release is metadata only, ~50% of files)", "sizeTier": "large", "licence": "MIT", "licenceUrl": "https://opensource.org/license/mit", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "OpenCoder-LLM (INF)", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "code", "pretraining", "reproducible", "multilingual", "large-corpus" ], "description": "A reproducible code pretraining corpus of roughly 960 billion tokens spanning about 607 programming languages, built with a published cleaning pipeline. It is the pretraining data behind the OpenCoder code models.", "links": { "documentation": "https://opencoder-llm.github.io/", "github": "https://github.com/OpenCoder-llm/OpenCoder-llm", "huggingface": "https://huggingface.co/datasets/OpenCoder-LLM/RefineCode-code-corpus-meta" } }, { "slug": "refinedweb", "name": "RefinedWeb", "page": "https://rag-repo.org/source/refinedweb", "url": "https://huggingface.co/datasets/tiiuae/falcon-refinedweb", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "600B tokens (public subset)", "sizeTier": "medium", "licence": "ODC-By 1.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Technology Innovation Institute (TII)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "english", "pretraining", "cleaned", "deduplicated", "falcon" ], "description": "A filtered English web dataset from the Technology Innovation Institute, creators of the Falcon models. It showed that carefully cleaned web-only data can match curated multi-source corpora for language model training.", "links": { "huggingface": "https://huggingface.co/datasets/tiiuae/falcon-refinedweb" } }, { "slug": "regulations-gov", "name": "Regulations.gov", "page": "https://rag-repo.org/source/regulations-gov", "url": "https://www.regulations.gov/", "category": "government", "categoryName": "Government & public sector", "accessType": "open", "accessPatterns": [ "api" ], "ragReadiness": "raw", "format": [ "JSON" ], "size": "1M+ documents, 20M+ public comments", "sizeTier": "unknown", "licence": "Public domain (U.S. Government works)", "licenceUrl": "https://www.usa.gov/government-works", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "U.S. eRulemaking Program (GSA)", "updateFrequency": null, "yearLaunched": 2003, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "us", "government", "legal", "public-comments", "api", "rulemaking" ], "description": "The official US federal rulemaking portal, exposing a public API to dockets, proposed and final rules, supporting materials and millions of public comments across federal agencies. Free to access with a registered API key.", "links": { "documentation": "https://open.gsa.gov/api/regulationsgov/", "api": "https://api.data.gov/regulations/v4/" } }, { "slug": "roots", "name": "ROOTS", "page": "https://rag-repo.org/source/roots", "url": "https://huggingface.co/bigscience-data", "category": "multilingual", "categoryName": "Multilingual & regional corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": "1.6 TB", "sizeTier": "large", "licence": "BigScience RAIL (varies by subset)", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "BigScience", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "multilingual", "bigscience", "bloom", "code", "pretraining" ], "description": "The Responsible Open-Science Open-Collaboration Text Sources corpus, a 1.6 TB dataset spanning 46 natural languages and 13 programming languages, released by BigScience. Built from community-selected sources plus OSCAR, it trained BLOOM.", "links": {} }, { "slug": "rteb", "name": "RTEB (Retrieval Embedding Benchmark)", "page": "https://rag-repo.org/source/rteb", "url": "https://huggingface.co/blog/rteb", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "chunked", "format": [], "size": "20 languages across open and private datasets", "sizeTier": "small", "licence": "Apache 2.0 (mteb framework); individual datasets vary, private sets held out", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Hugging Face and the MTEB community", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "retrieval", "embeddings", "benchmark", "evaluation", "multilingual", "leaderboard" ], "description": "The MTEB team's retrieval-focused embedding benchmark, launched in beta in October 2025 as a new retrieval section of the MTEB leaderboard. It spans 20 languages and enterprise domains such as law, healthcare, finance and code, and deliberately mixes open datasets with held-out private ones to measure genuine generalisation rather than training-set memorisation.", "links": { "github": "https://github.com/embeddings-benchmark/mteb", "huggingface": "https://huggingface.co/spaces/mteb/leaderboard" } }, { "slug": "saylor-academy", "name": "Saylor Academy", "page": "https://rag-repo.org/source/saylor-academy", "url": "https://www.saylor.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "web-crawl" ], "ragReadiness": "raw", "format": [ "HTML", "PDF" ], "size": "300+ courses", "sizeTier": "small", "licence": "CC BY 3.0 (course structure); embedded third-party materials vary", "licenceUrl": "https://www.saylor.org/open/licensing-information/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Saylor Academy", "updateFrequency": null, "yearLaunched": 1999, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "education", "open-courseware", "curriculum-aligned", "cc-by", "nonprofit", "self-paced", "higher-education" ], "description": "A nonprofit offering more than 300 free, self-paced college-level online courses across many subjects, with curated readings, framing text, and assessments. Course outlines are CC BY licensed; embedded third-party materials keep their own varying licences.", "links": { "documentation": "https://www.saylor.org/open/" } }, { "slug": "sdmx", "name": "SDMX (Statistical Data and Metadata Exchange)", "page": "https://rag-repo.org/source/sdmx", "url": "https://sdmx.org", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "api" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Open standard", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "standard", "statistics", "metadata", "interchange", "api" ], "description": "An open standard for exchanging statistical data and metadata electronically, developed jointly by the BIS, ECB, Eurostat, IMF, OECD, UN, and World Bank. Not a dataset itself but the interchange format many statistical agencies use to publish their figures.", "links": {} }, { "slug": "semantic-scholar", "name": "Semantic Scholar / S2ORC", "page": "https://rag-repo.org/source/semantic-scholar", "url": "https://www.semanticscholar.org", "category": "academic", "categoryName": "Academic & scientific literature", "accessType": "open", "accessPatterns": [ "huggingface", "api" ], "ragReadiness": "cleaned", "format": [ "JSON", "JSONL" ], "size": "Hundreds of millions of papers", "sizeTier": "large", "licence": "ODC-By (S2ORC); API terms for live access", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Allen Institute for AI", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "academic", "citations", "full-text", "nonprofit", "research", "api" ], "description": "Allen AI's academic graph, covering hundreds of millions of papers linked by billions of citations. S2ORC, the Semantic Scholar Open Research Corpus, is the downloadable dataset with full text and parsed references, drawn from arXiv, PubMed, Crossref, publishers, and web crawlers.", "links": { "huggingface": "https://huggingface.co/datasets/allenai/s2orc" } }, { "slug": "siyavula", "name": "Siyavula", "page": "https://rag-repo.org/source/siyavula", "url": "https://www.siyavula.com", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "cleaned", "format": [ "PDF", "EPUB", "HTML" ], "size": null, "sizeTier": "tiny", "licence": "CC BY-ND 4.0 (branded); CC BY (unbranded editions)", "licenceUrl": "https://creativecommons.org/licenses/by-nd/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Siyavula Education", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "textbooks", "mathematics", "physical-sciences", "k-12", "south-africa", "caps", "open-textbooks" ], "description": "Free, CAPS-aligned Everything Maths and Everything Science textbooks from Siyavula, covering South African school grades. Read online or download as PDF and EPUB, openly licensed for copying and redistribution.", "links": {} }, { "slug": "slimpajama", "name": "SlimPajama", "page": "https://rag-repo.org/source/slimpajama", "url": "https://huggingface.co/datasets/cerebras/SlimPajama-627B", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSONL" ], "size": "627B tokens", "sizeTier": "medium", "licence": "Apache 2.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Cerebras", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "pretraining", "cleaned", "deduplicated", "multi-source" ], "description": "A cleaned and deduplicated version of RedPajama-V1 from Cerebras. It drops short documents and removes near-duplicate content, cutting 1.2T tokens down to a denser 627B tokens without losing coverage.", "links": { "huggingface": "https://huggingface.co/datasets/cerebras/SlimPajama-627B" } }, { "slug": "smithsonian-open-access", "name": "Smithsonian Open Access", "page": "https://rag-repo.org/source/smithsonian-open-access", "url": "https://www.si.edu/openaccess", "category": "cultural-heritage", "categoryName": "Cultural heritage & archives", "accessType": "open", "accessPatterns": [ "api", "git-repo" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "unknown", "licence": "CC0 1.0", "licenceUrl": "https://creativecommons.org/publicdomain/zero/1.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Smithsonian Institution", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "cultural-heritage", "us", "museums", "cc0", "images", "3d-models" ], "description": "Millions of digital items from the Smithsonian's museums, research centres, and archives, released under CC0. Includes images, 3D models, and research datasets you can reuse freely, including commercially.", "links": {} }, { "slug": "snowflake-marketplace", "name": "Snowflake Marketplace", "page": "https://rag-repo.org/source/snowflake-marketplace", "url": "https://www.snowflake.com/en/product/features/marketplace/", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "commercial", "accessPatterns": [ "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Varies by listing", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Snowflake", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "data-platform", "marketplace", "commercial", "financial", "weather", "demographic" ], "description": "A data marketplace connecting more than 820 providers who offer over 3,400 live datasets, data services, and applications. Listings span financial, weather, demographic, and industry data, with a mix of free and paid options delivered straight into a Snowflake account.", "links": {} }, { "slug": "software-heritage", "name": "Software Heritage", "page": "https://rag-repo.org/source/software-heritage", "url": "https://www.softwareheritage.org", "category": "code", "categoryName": "Code & technical documentation", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [], "size": "27 B+ unique source files", "sizeTier": "huge", "licence": "Original source code licences apply; archive infrastructure is open source", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "Software Heritage", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "source-code", "archive", "preservation", "nonprofit", "software" ], "description": "A non-profit universal archive built specifically for software source code, preserving not just files but their full development history. Over 27 billion unique source files collected from public repositories, each identified by a cryptographic hash. The upstream source for The Stack v2.", "links": {} }, { "slug": "stack-exchange", "name": "Stack Exchange Data Dump", "page": "https://rag-repo.org/source/stack-exchange", "url": "https://archive.org/details/stackexchange", "category": "code", "categoryName": "Code & technical documentation", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [ "XML" ], "size": null, "sizeTier": "medium", "licence": "CC BY-SA 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Stack Exchange", "updateFrequency": "Quarterly", "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "question-answering", "stack-overflow", "community", "technical", "cc-by-sa" ], "description": "Periodic XML dumps of every site in the Stack Exchange network, including Stack Overflow. Each dump holds questions, answers, comments, votes, and user profiles. One of the richest question-and-answer datasets you can freely download, and a natural fit for building technical support and developer RAG systems.", "links": { "download": "https://archive.org/details/stackexchange" } }, { "slug": "stackmathqa", "name": "StackMathQA", "page": "https://rag-repo.org/source/stackmathqa", "url": "https://huggingface.co/datasets/math-ai/StackMathQA", "category": "mathematics", "categoryName": "Mathematics & reasoning", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "chunked", "format": [ "Parquet" ], "size": "2 M question-and-answer pairs", "sizeTier": "small", "licence": "CC BY-SA 4.0", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "mathematics", "question-answer", "stack-exchange", "qa" ], "description": "A curated collection of 2 million mathematical questions and answers sourced from various Stack Exchange sites. The question-and-answer structure maps naturally onto retrieval, which makes it well suited to Q&A-style RAG.", "links": {} }, { "slug": "standardized-gutenberg-corpus", "name": "Standardized Project Gutenberg Corpus (SPGC)", "page": "https://rag-repo.org/source/standardized-gutenberg-corpus", "url": "https://github.com/pgcorpus/gutenberg", "category": "books-literature", "categoryName": "Books & literature", "accessType": "open", "accessPatterns": [ "git-repo", "web-portal" ], "ragReadiness": "cleaned", "format": [ "TXT", "CSV" ], "size": null, "sizeTier": "small", "licence": "Public domain (content); MIT (code)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "public-domain", "books", "literature", "research", "cleaned" ], "description": "A research-ready version of the Project Gutenberg catalogue with consistent formatting, tidy metadata, and token counts for every book. It saves you the cleanup work, so you get uniform full-text literature ready to chunk and embed for RAG.", "links": { "github": "https://github.com/pgcorpus/gutenberg" } }, { "slug": "structured-wikipedia", "name": "Structured Wikipedia", "page": "https://rag-repo.org/source/structured-wikipedia", "url": "https://enterprise.wikimedia.com/api/structured-contents/", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "huggingface", "api" ], "ragReadiness": "cleaned", "format": [ "JSON", "Parquet" ], "size": "~44 GB (English and French, on Hugging Face)", "sizeTier": "small", "licence": "CC BY-SA 4.0 (text); GFDL (some content)", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Wikimedia Foundation", "updateFrequency": "Periodic snapshots", "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "encyclopaedia", "structured", "json", "nonprofit", "knowledge-graph", "pre-parsed" ], "description": "Wikipedia rendered as pre-parsed, machine-readable JSON: abstracts, short descriptions, infoboxes, sections, parsed tables and references, with links to Wikidata entities. A beta from Wikimedia Enterprise covering nine languages, also mirrored on Hugging Face. The section-segmented shape a RAG pipeline actually wants.", "links": { "huggingface": "https://huggingface.co/datasets/wikimedia/structured-wikipedia" } }, { "slug": "synth-pleias", "name": "SYNTH", "page": "https://rag-repo.org/source/synth-pleias", "url": "https://huggingface.co/datasets/PleIAs/SYNTH", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "Parquet" ], "size": "236 GB (~75B tokens)", "sizeTier": "medium", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "PleIAs", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "synthetic", "reasoning-traces", "multilingual", "encyclopaedic", "rag", "fine-tuning", "permissive-licence" ], "description": "A fully open synthetic corpus of amplified multilingual encyclopaedic text with built-in reasoning traces and exercises covering RAG, information extraction and QA. Released by PleIAs with the AI Alliance, it targets training and evaluating small, grounded, citeable reasoning models.", "links": { "documentation": "https://huggingface.co/datasets/PleIAs/SYNTH/blob/main/README.md", "huggingface": "https://huggingface.co/datasets/PleIAs/SYNTH" } }, { "slug": "t2-ragbench", "name": "T²-RAGBench", "page": "https://rag-repo.org/source/t2-ragbench", "url": "https://huggingface.co/datasets/G4KMU/t2-ragbench", "category": "rag-benchmarks", "categoryName": "RAG-specific & evaluation", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet", "JSON" ], "size": "23,088 QA pairs over 7,300+ documents", "sizeTier": "tiny", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "University of Hamburg and dida Datenschmiede GmbH", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "finance", "question-answering", "retrieval-benchmark", "tables", "numerical-reasoning", "sec-filings" ], "description": "A 2025 benchmark of 23,088 context-independent question, context and answer triples over more than 7,300 financial documents that mix text and tables. Each question maps to exactly one ground-truth document, making it purpose-built for numerical, table-aware financial RAG.", "links": { "github": "https://github.com/uhh-hcds/g4kmu-paper", "huggingface": "https://huggingface.co/datasets/G4KMU/t2-ragbench" } }, { "slug": "peoples-speech", "name": "The People's Speech", "page": "https://rag-repo.org/source/peoples-speech", "url": "https://huggingface.co/datasets/MLCommons/peoples_speech", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "FLAC", "JSON" ], "size": "30,000+ hours", "sizeTier": "large", "licence": "CC BY-SA 4.0 / CC BY 4.0 (subsets)", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "MLCommons", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "speech", "english", "asr", "commercial-friendly", "mlcommons" ], "description": "A large, freely licensed English speech recognition dataset from MLCommons, assembled from openly licensed sources and totalling more than 30,000 hours. Built as a permissively licensed alternative for training commercial automatic speech recognition systems.", "links": { "huggingface": "https://huggingface.co/datasets/MLCommons/peoples_speech" } }, { "slug": "the-pile", "name": "The Pile", "page": "https://rag-repo.org/source/the-pile", "url": "https://pile.eleuther.ai", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "mixed", "format": [ "JSONL" ], "size": "825 GB", "sizeTier": "medium", "licence": "MIT (compilation); individual subset licences vary", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "EleutherAI", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "pretraining", "multi-source", "english", "eleutherai", "academic" ], "description": "An 825 GB curated English text dataset from EleutherAI, made of 22 sub-datasets spanning books, academic papers, code, web content, and more. Built as a diverse training corpus and widely used to train early open language models.", "links": { "download": "https://pile.eleuther.ai" } }, { "slug": "the-stack", "name": "The Stack v1 / v2", "page": "https://rag-repo.org/source/the-stack", "url": "https://huggingface.co/datasets/bigcode/the-stack-v2", "category": "code", "categoryName": "Code & technical documentation", "accessType": "open", "accessPatterns": [ "huggingface", "cloud-bucket" ], "ragReadiness": "raw", "format": [ "Parquet" ], "size": null, "sizeTier": "large", "licence": "Permissive open-source licences only (as licensed by original repos)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "BigCode", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": "v1", "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": "the-stack-v2", "tags": [ "source-code", "permissive-licence", "pretraining", "bigcode", "deduplication", "multilingual-code" ], "description": "A large source code dataset from the BigCode project, built from permissively licensed code on GitHub with duplicate and near-duplicate files removed. Version 2 draws from Software Heritage and covers 600+ programming languages, making it a strong base for training and retrieval in code-aware RAG systems.", "links": { "huggingface": "https://huggingface.co/datasets/bigcode/the-stack-v2" } }, { "slug": "the-stack-v2", "name": "The Stack v2", "page": "https://rag-repo.org/source/the-stack-v2", "url": "https://huggingface.co/datasets/bigcode/the-stack-v2", "category": "code", "categoryName": "Code & technical documentation", "accessType": "open", "accessPatterns": [ "huggingface", "cloud-bucket" ], "ragReadiness": "raw", "format": [ "Parquet" ], "size": "67.5TB full (uncompressed); ~32.1TB deduplicated; ~900B tokens", "sizeTier": "huge", "licence": "Permissively licensed code only (gated access; files keep their original upstream licences)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "BigCode", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": "v2", "lastReviewed": "2026-09-10", "supersedes": [ "the-stack" ], "supersededBy": null, "tags": [ "source-code", "permissive-licence", "pretraining", "bigcode", "software-heritage", "starcoder2", "multilingual-code" ], "description": "BigCode's large-scale source code corpus and the pretraining set behind StarCoder2, built with Software Heritage and spanning more than 600 programming languages. The full version runs to about 67.5TB, with records pointing to code held in Software Heritage's S3 rather than embedding it directly.", "links": { "github": "https://github.com/bigcode-project/the-stack-v2", "huggingface": "https://huggingface.co/datasets/bigcode/the-stack-v2" } }, { "slug": "toolbench", "name": "ToolBench", "page": "https://rag-repo.org/source/toolbench", "url": "https://github.com/OpenBMB/ToolBench", "category": "agentic-tool-use", "categoryName": "Agentic & Tool-Use", "accessType": "open", "accessPatterns": [ "git-repo", "bulk-download" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": "126,486 instruction instances over 16,464 APIs", "sizeTier": "small", "licence": "Apache 2.0 (research and educational use)", "licenceUrl": "https://www.apache.org/licenses/LICENSE-2.0", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "OpenBMB", "updateFrequency": null, "yearLaunched": 2023, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "agentic", "tool-use", "function-calling", "instruction-tuning", "fine-tuning", "rest-api", "research" ], "description": "An open instruction-tuning dataset for teaching general tool use to language models, built for the ToolLLM project over 16,464 real-world REST APIs from RapidAPI across 49 categories, with single-tool and multi-tool, multi-step solution paths.", "links": { "github": "https://github.com/OpenBMB/ToolBench" } }, { "slug": "toucan-agentic", "name": "Toucan-1.5M", "page": "https://rag-repo.org/source/toucan-agentic", "url": "https://huggingface.co/datasets/Agent-Ark/Toucan-1.5M", "category": "agentic-tool-use", "categoryName": "Agentic & Tool-Use", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~1.65M trajectories", "sizeTier": "small", "licence": "Apache 2.0", "licenceUrl": "https://www.apache.org/licenses/LICENSE-2.0", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Agent-Ark", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "agentic", "tool-use", "mcp", "function-calling", "synthetic", "fine-tuning", "multi-turn" ], "description": "The largest open tool-agentic dataset: over 1.5 million trajectories synthesised from 495 real-world MCP servers spanning 2,000 plus tools, with multi-turn, sequential and parallel tool calls backed by real executions and error handling. Built by Agent-Ark and released under Apache 2.0, it is premier open data for training retrieval-and-tool (MCP) agents.", "links": { "github": "https://github.com/TheAgentArk/Toucan", "huggingface": "https://huggingface.co/datasets/Agent-Ark/Toucan-1.5M" } }, { "slug": "txt360", "name": "TxT360", "page": "https://rag-repo.org/source/txt360", "url": "https://huggingface.co/datasets/LLM360/TxT360", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "JSONL" ], "size": "15 T+ tokens (~23.9 TB)", "sizeTier": "large", "licence": "ODC-By 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "LLM360", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "pretraining", "deduplicated", "large-corpus", "reproducible" ], "description": "An open pre-training corpus of 15 trillion-plus tokens from LLM360, built by globally deduplicating 99 Common Crawl snapshots and blending in 14 curated domains such as FreeLaw, PG-19, Wikipedia, and scientific papers. Ships with a fully documented, reproducible processing recipe.", "links": { "github": "https://github.com/LLM360/TxT360", "huggingface": "https://huggingface.co/datasets/LLM360/TxT360" } }, { "slug": "uk-data-service", "name": "UK Data Service", "page": "https://rag-repo.org/source/uk-data-service", "url": "https://ukdataservice.ac.uk", "category": "curated-lists", "categoryName": "Curated lists & meta-resources", "accessType": "limited", "accessPatterns": [ "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "Check source", "licenceUrl": null, "commercialUse": "restricted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "UK Data Service (ESRC)", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "uk", "social-science", "research", "registration-required", "academic" ], "description": "The UK's largest collection of economic, population, and social research data for teaching, learning, and public benefit. Many datasets need registration or an institutional login, so plan for an access step before you build with them.", "links": { "documentation": "https://ukdataservice.ac.uk" } }, { "slug": "uk-national-curriculum", "name": "UK National Curriculum (England)", "page": "https://rag-repo.org/source/uk-national-curriculum", "url": "https://www.gov.uk/government/collections/national-curriculum", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "raw", "format": [ "HTML", "PDF" ], "size": null, "sizeTier": "tiny", "licence": "Crown copyright, Open Government Licence v3.0", "licenceUrl": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Department for Education", "updateFrequency": "Periodic", "yearLaunched": 2014, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "curriculum", "uk", "standards", "k12", "government", "open-government-licence" ], "description": "The statutory programmes of study and attainment targets for state schools in England, organised by key stage and subject and published by the Department for Education. The curriculum spine for England, freely reusable under the Open Government Licence.", "links": {} }, { "slug": "un-data", "name": "UN Data", "page": "https://rag-repo.org/source/un-data", "url": "https://data.un.org", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "web-portal" ], "ragReadiness": "cleaned", "format": [], "size": "60M+ records across 32 databases", "sizeTier": "unknown", "licence": "UN terms (generally open)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "United Nations Statistics Division", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "statistics", "un", "global", "development", "trade", "energy" ], "description": "A single access point to the UN system's statistical databases: 32 databases holding over 60 million records from 1970 onward. Covers greenhouse gas inventories, commodity trade statistics, energy statistics, gender, and key global development indicators.", "links": {} }, { "slug": "uniprot", "name": "UniProt", "page": "https://rag-repo.org/source/uniprot", "url": "https://www.uniprot.org", "category": "biomedical", "categoryName": "Biomedical & health", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "FASTA", "XML", "JSON", "RDF" ], "size": "250M+ protein sequences", "sizeTier": "medium", "licence": "CC BY 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "UniProt Consortium", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "proteins", "sequences", "bioinformatics", "structured", "api", "medical-research" ], "description": "A resource for protein sequence and functional information, with more than 250 million protein sequences and rich annotations. It is a core reference for bioinformatics, so it grounds life sciences RAG systems in reliable protein facts.", "links": { "api": "https://www.uniprot.org/help/api" } }, { "slug": "unpaywall", "name": "Unpaywall", "page": "https://rag-repo.org/source/unpaywall", "url": "https://unpaywall.org", "category": "academic", "categoryName": "Academic & scientific literature", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [ "JSON" ], "size": "30M+ articles", "sizeTier": "small", "licence": "CC0 (data); open-source tools", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": "OurResearch", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "open-access", "doi", "metadata", "nonprofit", "research", "api" ], "description": "A database that tracks where scholarly articles can be read legally for free, covering over 30 million open-access papers. Maintained by the nonprofit OurResearch, it links each article's DOI to full-text versions hosted across repositories and journals.", "links": {} }, { "slug": "upstash-wikipedia-bge-m3", "name": "Upstash Wikipedia 2024 BGE-M3 Embeddings", "page": "https://rag-repo.org/source/upstash-wikipedia-bge-m3", "url": "https://huggingface.co/datasets/Upstash/wikipedia-2024-06-bge-m3", "category": "embedded-datasets", "categoryName": "Pre-embedded & RAG-ready", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "embedded", "format": [ "Parquet" ], "size": "~144 million 1024-dimensional vectors", "sizeTier": "medium", "licence": "Apache 2.0 (dataset); CC BY-SA (Wikipedia text)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Upstash", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": "2024-06", "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "wikipedia", "embeddings", "bge-m3", "multilingual", "semantic-search", "rag-ready", "parquet" ], "description": "The June 2024 Wikipedia dump split into paragraphs and pre-embedded with the multilingual BGE-M3 model, roughly 144 million vectors across the 11 most popular languages. Each paragraph is prefixed with its article title before embedding and very short paragraphs are dropped, so you can skip the embedding step and load meaning-ready vectors straight into a vector database.", "links": { "huggingface": "https://huggingface.co/datasets/Upstash/wikipedia-2024-06-bge-m3" } }, { "slug": "vidore-benchmark", "name": "ViDoRe (Visual Document Retrieval Benchmark)", "page": "https://rag-repo.org/source/vidore-benchmark", "url": "https://github.com/illuin-tech/vidore-benchmark", "category": "retrieval-benchmarks", "categoryName": "Retrieval benchmarks & evaluation", "accessType": "open", "accessPatterns": [ "huggingface", "git-repo" ], "ragReadiness": "mixed", "format": [ "Images", "Parquet" ], "size": "~10 page-level retrieval tasks, several thousand document page images", "sizeTier": "small", "licence": "Varies (evaluation code MIT; task datasets under their own licences)", "licenceUrl": "https://github.com/illuin-tech/vidore-benchmark/blob/main/LICENSE", "commercialUse": "unclear", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "ILLUIN Technology", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-08-17", "supersedes": [], "supersededBy": null, "tags": [ "retrieval-benchmark", "document-retrieval", "multimodal", "vision-language", "multilingual", "evaluation" ], "description": "A benchmark for OCR-free, page-image document retrieval, introduced with the ColPali paper. It bundles page-level retrieval tasks across several domains and languages so vision-based retrievers can be evaluated directly on document images.", "links": { "documentation": "https://arxiv.org/abs/2407.01449", "github": "https://github.com/illuin-tech/vidore-benchmark", "huggingface": "https://huggingface.co/vidore" } }, { "slug": "voxceleb", "name": "VoxCeleb", "page": "https://rag-repo.org/source/voxceleb", "url": "https://www.robots.ox.ac.uk/~vgg/data/voxceleb/", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "limited", "accessPatterns": [ "bulk-download", "web-crawl" ], "ragReadiness": "raw", "format": [ "WAV" ], "size": "100,000+ utterances from 7,000+ speakers", "sizeTier": "medium", "licence": "CC BY 4.0 (annotations); source video terms apply", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Visual Geometry Group, University of Oxford", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "speech", "speaker-recognition", "youtube", "voice", "research" ], "description": "An audio dataset of more than 100,000 utterances from over 7,000 speakers, extracted from interview videos uploaded to YouTube. Widely used for speaker recognition and verification. The annotations are CC BY 4.0, but the terms of the underlying source videos still apply.", "links": {} }, { "slug": "weatherbench-2", "name": "WeatherBench 2", "page": "https://rag-repo.org/source/weatherbench-2", "url": "https://sites.research.google/weatherbench/", "category": "climate-earth", "categoryName": "Climate & earth observation", "accessType": "open", "accessPatterns": [ "cloud-bucket" ], "ragReadiness": "raw", "format": [ "Zarr" ], "size": null, "sizeTier": "large", "licence": "Benchmark code Apache-2.0; ERA5 ground-truth data under the Copernicus/ECMWF licence", "licenceUrl": "https://www.apache.org/licenses/LICENSE-2.0", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Google Research", "updateFrequency": "Ongoing", "yearLaunched": 2023, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "weather-forecasting", "benchmark", "era5", "climate", "evaluation", "machine-learning", "google" ], "description": "Google Research's open benchmark and curated ERA5-derived data archive for evaluating data-driven global weather models over the one-to-fifteen-day range. It bundles an Apache-licensed evaluation framework, cloud-hosted ground-truth and baseline forecast datasets in Zarr, and a continuously updated leaderboard ranking models such as GraphCast and Pangu-Weather against ECMWF's IFS.", "links": { "documentation": "https://weatherbench2.readthedocs.io/", "github": "https://github.com/google-research/weatherbench2" } }, { "slug": "wikidata", "name": "Wikidata", "page": "https://rag-repo.org/source/wikidata", "url": "https://www.wikidata.org", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "mixed", "format": [ "JSON", "RDF" ], "size": "100M+ items", "sizeTier": "medium", "licence": "CC0 1.0 (public domain)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Wikimedia Foundation", "updateFrequency": null, "yearLaunched": 2012, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "knowledge-base", "structured", "multilingual", "public-domain", "wikimedia", "identifiers" ], "description": "A free, collaborative knowledge base that holds structured facts for Wikipedia and the other Wikimedia projects. More than 100M items, each one machine-readable with statements, relationships, and identifiers, so you can pull clean facts instead of parsing article prose. Released into the public domain under CC0.", "links": { "api": "https://www.wikidata.org/w/api.php" } }, { "slug": "wikidata-embedding-project", "name": "Wikidata Embedding Project", "page": "https://rag-repo.org/source/wikidata-embedding-project", "url": "https://www.wikidata.org/wiki/Wikidata:Embedding_Project", "category": "embedded-datasets", "categoryName": "Pre-embedded & RAG-ready", "accessType": "open", "accessPatterns": [ "api" ], "ragReadiness": "embedded", "format": [ "REST API", "JSON", "MCP" ], "size": "Nearly 120 million items", "sizeTier": "unknown", "licence": "CC0 (structured data); CC BY-SA (textual content)", "licenceUrl": "https://www.wikidata.org/wiki/Wikidata:Licensing", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Wikimedia Deutschland", "updateFrequency": null, "yearLaunched": 2025, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "wikidata", "embeddings", "knowledge-graph", "semantic-search", "mcp", "multilingual", "structured-data" ], "description": "A hosted semantic-search service over Wikidata from Wikimedia Deutschland, built with Jina.AI and DataStax, serving vector embeddings of nearly 120 million items. You query it by meaning through a REST API and a native Model Context Protocol endpoint rather than downloading files, with reranking and structured context returned for each hit.", "links": { "documentation": "https://wd-vectordb.wmcloud.org/docs", "api": "https://wd-vectordb.wmcloud.org" } }, { "slug": "wikimedia-commons", "name": "Wikimedia Commons", "page": "https://rag-repo.org/source/wikimedia-commons", "url": "https://commons.wikimedia.org", "category": "cultural-heritage", "categoryName": "Cultural heritage & archives", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "raw", "format": [], "size": "100M+ media files", "sizeTier": "huge", "licence": "CC0 / CC BY-SA (varies by file)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Wikimedia Foundation", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "media", "images", "freely-licensed", "wikimedia", "structured-data", "public-domain" ], "description": "A media repository that hosts most of the images, video, audio, and other files used across Wikimedia projects. Over 100 million freely licensed files, with structured metadata through Structured Data on Commons, usable by anyone for almost any purpose.", "links": {} }, { "slug": "wikipedia", "name": "Wikipedia", "page": "https://rag-repo.org/source/wikipedia", "url": "https://dumps.wikimedia.org", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "bulk-download", "huggingface" ], "ragReadiness": "raw", "format": [ "XML", "SQL" ], "size": "English text dump is ~22 GB compressed", "sizeTier": "small", "licence": "CC BY-SA 4.0 (text); GFDL (some content)", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Wikimedia Foundation", "updateFrequency": "Twice monthly", "yearLaunched": 2001, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "encyclopaedia", "multilingual", "general-knowledge", "nonprofit", "bulk-download" ], "description": "Complete database dumps of Wikipedia and the other Wikimedia projects, in every language, refreshed roughly twice a month. The most widely used knowledge source for RAG systems, and an easy first corpus to build a retrieval pipeline on.", "links": { "documentation": "https://meta.wikimedia.org/wiki/Data_dumps", "download": "https://dumps.wikimedia.org", "huggingface": "https://huggingface.co/datasets/wikimedia/wikipedia" } }, { "slug": "wikiversity", "name": "Wikiversity", "page": "https://rag-repo.org/source/wikiversity", "url": "https://www.wikiversity.org", "category": "education", "categoryName": "Education & open learning", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "raw", "format": [ "XML", "HTML", "PDF" ], "size": null, "sizeTier": "tiny", "licence": "CC BY-SA 4.0 (text); some GFDL content", "licenceUrl": "https://creativecommons.org/licenses/by-sa/4.0/", "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Wikimedia Foundation", "updateFrequency": "Continuous (dumps refreshed roughly twice monthly)", "yearLaunched": 2006, "isArchived": false, "version": null, "lastReviewed": "2026-09-15", "supersedes": [], "supersededBy": null, "tags": [ "open-educational-resources", "wikimedia", "cc-by-sa", "lessons", "multilingual", "bulk-download", "nonprofit" ], "description": "Free, collaboratively written learning materials from the Wikimedia Foundation: lessons, courses and tutorials organised by subject and educational level, in many languages. Available as bulk dumps and via API, like the other Wikimedia projects.", "links": { "documentation": "https://meta.wikimedia.org/wiki/Data_dumps", "download": "https://dumps.wikimedia.org" } }, { "slug": "wipo-patentscope", "name": "WIPO PATENTSCOPE", "page": "https://rag-repo.org/source/wipo-patentscope", "url": "https://www.wipo.int/patentscope/en/", "category": "patents", "categoryName": "Patents & intellectual property", "accessType": "open", "accessPatterns": [ "web-portal" ], "ragReadiness": "raw", "format": [], "size": null, "sizeTier": "unknown", "licence": "WIPO terms", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "World Intellectual Property Organization", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "patents", "global", "multilingual", "search", "wipo" ], "description": "A search service for international patent collections, maintained by the World Intellectual Property Organization (WIPO). It lets you search across patent applications filed under international treaties alongside national collections from many countries.", "links": {} }, { "slug": "wit", "name": "WIT (Wikipedia-based Image Text)", "page": "https://rag-repo.org/source/wit", "url": "https://github.com/google-research-datasets/wit", "category": "multimodal", "categoryName": "Multimodal & image-text", "accessType": "open", "accessPatterns": [ "git-repo", "huggingface", "web-portal" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "small", "licence": "CC BY-SA 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": true, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Google Research", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "image-text", "multilingual", "wikipedia", "multimodal", "share-alike" ], "description": "A Wikipedia-based image-text dataset for multimodal, multilingual machine learning. It pairs images with their Wikipedia captions and the surrounding article text across many languages, giving richer context than a single caption.", "links": {} }, { "slug": "wordnet", "name": "WordNet", "page": "https://rag-repo.org/source/wordnet", "url": "https://wordnet.princeton.edu", "category": "knowledge-graphs", "categoryName": "Knowledge graphs & structured data", "accessType": "open", "accessPatterns": [ "bulk-download" ], "ragReadiness": "mixed", "format": [], "size": null, "sizeTier": "tiny", "licence": "Princeton WordNet License (permissive)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "Princeton University", "updateFrequency": null, "yearLaunched": 1985, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "lexical-database", "english", "synonyms", "nlp", "princeton" ], "description": "A lexical database of English that groups nouns, verbs, adjectives, and adverbs into sets of synonyms called synsets, then links those sets by meaning. It maps how words relate, which sense means what, what is a kind of what, so software can work with meaning rather than just spelling.", "links": { "download": "https://wordnet.princeton.edu/download" } }, { "slug": "world-bank-open-data", "name": "World Bank Open Data", "page": "https://rag-repo.org/source/world-bank-open-data", "url": "https://data.worldbank.org", "category": "statistics-economics", "categoryName": "Statistics & economics", "accessType": "open", "accessPatterns": [ "api", "bulk-download" ], "ragReadiness": "cleaned", "format": [], "size": "900+ development indicators, 210 countries", "sizeTier": "small", "licence": "CC BY 4.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "World Bank", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "development", "economics", "statistics", "time-series", "global", "api" ], "description": "Free and open access to global development data from the World Bank, including the World Development Indicators: more than 900 indicators with time series for 210 countries from 1960 to the present. Also includes a Microdata Library of sample survey data and a Data Catalog for bulk downloads.", "links": {} }, { "slug": "world-resources-institute", "name": "World Resources Institute", "page": "https://rag-repo.org/source/world-resources-institute", "url": "https://www.wri.org/data", "category": "climate-earth", "categoryName": "Climate & earth observation", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "cleaned", "format": [], "size": null, "sizeTier": "unknown", "licence": "CC BY 4.0 (mostly)", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "World Resources Institute", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "environment", "climate", "gis", "forests", "sustainability" ], "description": "Free environmental data for GIS, climate research, and sustainability projects. Includes Global Forest Watch and other environmental monitoring datasets.", "links": {} }, { "slug": "xlam-function-calling", "name": "xLAM Function-Calling (APIGen)", "page": "https://rag-repo.org/source/xlam-function-calling", "url": "https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k", "category": "agentic-tool-use", "categoryName": "Agentic & Tool-Use", "accessType": "limited", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "JSON" ], "size": "60,000 examples across 3,673 APIs", "sizeTier": "tiny", "licence": "CC BY 4.0", "licenceUrl": "https://creativecommons.org/licenses/by/4.0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Salesforce AI Research", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "agentic", "tool-use", "function-calling", "synthetic", "fine-tuning", "verified", "salesforce" ], "description": "Salesforce's APIGen-generated dataset of 60,000 verified function-calling examples spanning 3,673 executable APIs, each checked through format, execution and semantic stages. The training data behind the xLAM action models and core capability data for tool-using agents.", "links": { "documentation": "https://apigen-pipeline.github.io/", "github": "https://github.com/SalesforceAIResearch/xLAM", "huggingface": "https://huggingface.co/datasets/Salesforce/xlam-function-calling-60k" } }, { "slug": "yago", "name": "YAGO", "page": "https://rag-repo.org/source/yago", "url": "https://yago-knowledge.org", "category": "encyclopaedic", "categoryName": "Encyclopaedic & general knowledge", "accessType": "open", "accessPatterns": [ "bulk-download", "api" ], "ragReadiness": "mixed", "format": [ "RDF" ], "size": null, "sizeTier": "small", "licence": "CC BY 3.0", "licenceUrl": null, "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": false, "maintainer": "Max Planck Institute for Informatics and Télécom Paris", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "knowledge-base", "structured", "wikipedia", "wordnet", "geonames", "rdf" ], "description": "A semantic knowledge base that combines facts from Wikipedia, WordNet, and GeoNames into a single, high-accuracy collection of statements about entities. It adds when and where each fact holds, so you get temporal and spatial detail alongside the plain relationships.", "links": { "download": "https://yago-knowledge.org/downloads" } }, { "slug": "yfcc100m", "name": "YFCC100M", "page": "https://rag-repo.org/source/yfcc100m", "url": "https://multimediacommons.wordpress.com", "category": "multimodal", "categoryName": "Multimodal & image-text", "accessType": "open", "accessPatterns": [ "cloud-bucket" ], "ragReadiness": "raw", "format": [], "size": "100M items", "sizeTier": "large", "licence": "CC (varies by item)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": true, "maintainer": null, "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "image-text", "video", "flickr", "creative-commons", "multimodal", "metadata" ], "description": "A multimedia research dataset of 100M Flickr photos and videos with their metadata, all published under Creative Commons licences. The exact licence varies from one item to the next, so check each one before you reuse it.", "links": {} }, { "slug": "yodas2", "name": "YODAS2", "page": "https://rag-repo.org/source/yodas2", "url": "https://huggingface.co/datasets/espnet/yodas2", "category": "speech-audio", "categoryName": "Speech & audio", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "mixed", "format": [ "WAV", "JSON" ], "size": "60.3 TB (500,000+ hours, 149 languages)", "sizeTier": "huge", "licence": "CC BY 3.0 (per-video Creative Commons licences)", "licenceUrl": "https://creativecommons.org/licenses/by/3.0/", "commercialUse": "restricted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "ESPnet", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "speech", "multilingual", "audio", "youtube", "creative-commons", "speech-recognition", "long-form" ], "description": "YODAS2 is the long-form edition of ESPnet's YODAS corpus, offering over 500,000 hours of Creative Commons YouTube speech across 149 languages, re-released as full-video audio at 24 kHz. It carries both labelled subsets (manual and automatic subtitles) and unlabelled audio for self-supervised work.", "links": { "documentation": "https://arxiv.org/abs/2406.00899", "github": "https://github.com/espnet/espnet", "huggingface": "https://huggingface.co/datasets/espnet/yodas2" } }, { "slug": "zenodo", "name": "Zenodo", "page": "https://rag-repo.org/source/zenodo", "url": "https://zenodo.org", "category": "data-platforms", "categoryName": "Data platforms & marketplaces", "accessType": "open", "accessPatterns": [ "web-portal", "api" ], "ragReadiness": "mixed", "format": [], "size": null, "sizeTier": "unknown", "licence": "Varies (set by depositor)", "licenceUrl": null, "commercialUse": "unclear", "shareAlike": false, "attributionRequired": false, "contentLicenceDiffers": false, "maintainer": "CERN", "updateFrequency": null, "yearLaunched": null, "isArchived": false, "version": null, "lastReviewed": "2026-08-06", "supersedes": [], "supersededBy": null, "tags": [ "data-platform", "open-access", "research", "doi", "cern" ], "description": "A general-purpose open repository built by CERN where researchers can deposit datasets, software, reports, and any other digital output. Every upload gets a DOI, a permanent identifier that makes the work easy to cite and find again, which makes Zenodo a reliable long-term home for research data.", "links": { "api": "https://developers.zenodo.org" } }, { "slug": "zyda-2", "name": "Zyda-2", "page": "https://rag-repo.org/source/zyda-2", "url": "https://huggingface.co/datasets/Zyphra/Zyda-2", "category": "web-corpora", "categoryName": "Web corpora", "accessType": "open", "accessPatterns": [ "huggingface" ], "ragReadiness": "cleaned", "format": [ "Parquet" ], "size": "~13 TB (5T tokens)", "sizeTier": "large", "licence": "ODC-BY 1.0", "licenceUrl": "https://opendatacommons.org/licenses/by/1-0/", "commercialUse": "permitted", "shareAlike": false, "attributionRequired": true, "contentLicenceDiffers": true, "maintainer": "Zyphra", "updateFrequency": null, "yearLaunched": 2024, "isArchived": false, "version": null, "lastReviewed": "2026-09-10", "supersedes": [], "supersededBy": null, "tags": [ "web-crawl", "english", "pretraining", "filtered", "deduplicated", "huggingface", "zyphra" ], "description": "Zyphra's five trillion token English pretraining mixture, built by filtering and cross-deduplicating DCLM, FineWeb-Edu, Zyda-1 and Dolma's Common Crawl portion. A ready high-quality drop-in base that Zyphra reports beats its component datasets on downstream evaluation. Published on HuggingFace as Parquet under an open licence.", "links": { "huggingface": "https://huggingface.co/datasets/Zyphra/Zyda-2" } } ] }