{
    "domain_prefix": "lbty.org",
    "id": "117b6a7d-5126-41fe-9bc2-d276e98632e6",
    "meta": "{\"domain\": \"dclm\", \"ori_score\": 0.043276190757751465, \"source\": \"dclm_baseline\"}",
    "text": "Sabine Expedition\n\nThe Sabine Expedition was an expedition approved by the United States Congress in 1806...",
    "tokens": 145,  # Token count using Qwen2.5 tokenizer
    "url": "[https://lbty.org/american-indian-battles/sabine-expedition/](https://lbty.org/american-indian-battles/sabine-expedition/)",
    "score": 0.19072403013706207
}

from datasets import load_dataset

# Load the high-quality subset: medium-high < high < ultra
dataset = load_dataset("OpenSQZ/AutoMathText-V2", name="medium-high", streaming=True)
dataset = load_dataset("OpenSQZ/AutoMathText-V2", name="high", streaming=True)
dataset = load_dataset("OpenSQZ/AutoMathText-V2", name="ultra", streaming=True)

# Load specific domain
math_data = load_dataset("OpenSQZ/AutoMathText-V2", name="math_web", streaming=True)

import os
import json
import boto3
from smart_open import open
from datasets import load_dataset

# Setup AWS credentials
session = boto3.Session(
    aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
    aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"]
)
s3 = session.client("s3")

def download_code_content(blob_id, src_encoding):
    """Download code content from AWS S3 using blob_id"""
    s3_url = f"s3://softwareheritage/content/{blob_id}"
    
    try:
        with open(s3_url, "rb", compression=".gz", transport_params={"client": s3}) as fin:
            content = fin.read().decode(src_encoding)
        return {"content": content}
    except Exception as e:
        return {"content": None, "error": str(e)}

# Load RefineCode domain
refinecode_data = load_dataset("OpenSQZ/AutoMathText-V2", name="refinecode", streaming=True)

# Process each sample to download content
for sample in refinecode_data:
    # Parse metadata to extract blob_id and encoding
    meta = json.loads(sample["meta"])
    blob_id = meta.get("blob_id")
    src_encoding = meta.get("src_encoding", "utf-8")
    
    if blob_id:
        # Download the actual code content
        code_data = download_code_content(blob_id, src_encoding)
        
        # Combine metadata with downloaded content
        full_sample = {
            **sample,
            "code_content": code_data["content"]
        }
        
        print(f"Downloaded content for {sample['id']}")
        print(f"Content length: {len(code_data['content']) if code_data['content'] else 0}")
        break

AutoMathText-V2/
├── dclm/                  # DCLM baseline web content
│   ├── 0-10/             # Bottom 10% quality tokens (score-based)
│   ├── 10-20/            # 10-20% quality tokens
│   ├── 20-30/            # 20-30% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── fineweb_edu/           # FineWeb educational content
│   ├── 0-10/             # Bottom 10% quality tokens
│   ├── 10-20/            # 10-20% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── fineweb_edu_chinese/   # Chinese educational content
│   ├── 0-10/             # Bottom 10% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── math_web/              # Mathematics and scientific content
│   ├── 0-10/  .          # Bottom 10% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── megamath/              # Specialized math collections
│   ├── 0-10/             # Bottom 10% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── nemotron_cc_high/      # High quality Nemotron CommonCrawl
│   ├── 0-10/             # Bottom 10% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── nemotron_cc_medium_high/ # Medium-high quality Nemotron CommonCrawl
│   ├── 0-10/            . # Bottom 10% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── reasoning_qa/          # Instruction and reasoning data
│   ├── 0-10/             # Bottom 10% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
├── refinecode/            # GitHub code repositories (Academic Use Only)
│   ├── 0-10/             # Bottom 10% quality tokens
│   ├── ...               # Additional percentile ranges
│   └── 90-100/           # Top 10% highest quality tokens
└── translation/           # English-Chinese translation pairs
    ├── 0-10/             # Bottom 10% quality tokens
    ├── ...               # Additional percentile ranges
    └── 90-100/           # Top 10% highest quality tokens

@misc{automathtext_v2_2025,
  title={AutoMathText-V2: A 2.46 Trillion Token AI-Curated STEM Pretraining Dataset},
  author={Li, Chao and Zhang, Yifan and Lu, Yanzhen and Yao, Andrew C},
  year={2025},
  publisher={Hugging Face},
  url={https://huggingface.co/datasets/OpenSQZ/AutoMathText-V2},
  note={A 2.46T token multi-domain dataset with fine-grained deduplication and AI-powered quality assessment.}
}

@article{zhang2025autonomous,
  title={Autonomous Data Selection with Zero-shot Generative Classifiers for Mathematical Texts},
  author={Zhang, Yifan and Luo, Yifan and Yuan, Yang and Yao, Andrew C},
  journal={Findings of the Association for Computational Linguistics: ACL 2025},
  year={2025}
}