pandas-dev-pandas-66327-66275

347:        if _n_workers > 1 and _can_parallelize_csv(filepath_or_buffer, kwds):
349:            assert isinstance(_filepath, str)  # guaranteed by _can_parallelize_csv
351:                result = _read_csv_parallel(_filepath, kwds, _n_workers)
371:def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
540:    # enough to split into at least 2 chunks.  Without this, _read_csv_parallel
586:def _find_chunk_byte_offsets(
623:def _read_csv_parallel(
639:    The caller must ensure the file is eligible via :func:`_can_parallelize_csv`
661:    offsets = _find_chunk_byte_offsets(filepath, n_workers, data_start)

"The 'iterator' option is not supported with the 'pyarrow' engine"
            )

        if chunksize is not None:
            raise ValueError(
                "The 'chunksize' option is not supported with the 'pyarrow' engine"
            )
    else:
        chunksize = validate_integer("chunksize", chunksize, 1)

    nrows = kwds.get("nrows", None)

    # Check for duplicates in names.
    _validate_names(kwds.get("names", None))

    if kwds.get("float_precision") is not None:
        warnings.warn(
            "The 'float_precision' argument is deprecated. "
            "Use the default float precision instead.",
            Pandas4Warning,
            stacklevel=find_stack_level(),
        )

    # For large local uncompressed files with the C engine, attempt parallel reading.
    # Each worker gets its own file handle and TextReader so the GIL-free tokenisation
…[8629 chars omitted]…
if header is not None and isinstance(kwds.get("dtype"), dict):
        header_line = preamble.splitlines(keepends=True)[-1]
        raw_reader = TextFileReader(
            io.BytesIO(header_line),
            **{
                **base_kwds,
                "header": None,
                "names": None,
                "usecols": None,
                "index_col": False,
                "dtype": str,

def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
    """
    Return True when a ``read_csv`` call is eligible for parallel execution.

    Parallel reading works by splitting the file's data section into
    byte-range chunks at newline boundaries and processing them concurrently
    with threads.  The following conditions must all hold:

    * *filepath_or_buffer* is a local file path (not a URL or file object).
    * The file is not compressed.
    * The C engine is selected (``engine='c'``, the default).
    * The call is not in iterator / chunked mode.

    * The entire file is being read (no ``nrows`` limit).
    * ``skiprows`` is a plain integer (or absent) - list/callable forms require
      tracking absolute line numbers across chunks.
    * ``header`` is a single integer or ``None`` - multi-level headers complicate
      the preamble boundary.
    * ``index_co
…[7916 chars omitted]…
 is the byte from which each non-first chunk
    must begin.
    """
    if header is None:
        header_lines = 0
    else:
        header_lines = int(header) + 1  # header=0 → 1 header line

    preamble_lines = int(skiprows) + header_lines
    with open(filepath, "rb") as fd:
        for _ in range(preamble_lines):
            if not fd.readline():
                break
        return fd.tell()

185:_PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
408:    * The file is at least ``_PARALLEL_READ_MIN_BYTES`` bytes large.
534:        if file_size < _PARALLEL_READ_MIN_BYTES:
547:    if file_size - data_start < _PARALLEL_READ_MIN_BYTES:

148:        low_memory: bool
160:    low_memory: Literal[True]
167:    "low_memory": True,
181:_python_unsupported = {"low_memory", "float_precision"}
201:    "low_memory",
891:    low_memory: bool = _c_parser_defaults["low_memory"],
1225:    low_memory : bool, default True
1485:    low_memory: bool = _c_parser_defaults["low_memory"],
1815:    low_memory : bool, default True

"""
Tests for the parallel read_csv implementation (C engine, large local files).

The parallel path is enabled automatically when:
  - reading from a local uncompressed file path
  - using the C engine (the default)
  - not in iterator / chunked mode
  - file size >= _PARALLEL_READ_MIN_BYTES

We test correctness (parallel == serial) rather than performance.
"""

from __future__ import annotations

import csv
import io
import os
from typing import TYPE_CHECKING

import numpy as np
import pytest

from pandas.errors import (
    ParserError,
    ParserWarning,
)

if TYPE_CHECKING:
    from pathlib import Path

from pandas import (
    DataFrame,
    option_context,
    read_csv,
)
import pandas._testing as tm

from pandas.io.parsers.base_parser import ParserBase
from pandas.io.parsers.readers import (
    _can_parallelize_csv,
    _find_chunk_byte_offsets,
    _find_data_start_offset,
 
…[8629 chars omitted]…
nitialspace": False,
            "skiprows": None,
            "skipfooter": 0,
            "nrows": None,
            "na_values": None,
            "keep_default_na": True,
            "na_filter": True,
            "skip_blank_lines": True,
            "parse_dates": None,
            "date_format": None,
            "dayfirst": False,
            "cache_dates": True,
            "iterator": False,

# _read_csv_parallel  (correctness: parallel == serial)
# ---------------------------------------------------------------------------


@pytest.mark.slow
class TestReadCsvParallel:
    """End-to-end correctness: parallel result must match serial result."""

    def _serial_read(self, path, **kwargs):
        return read_csv(path, engine="c", **kwargs)

    def _parallel_read(self, path, kwds, n_workers=4):

        """Call the internal helper directly so file-size guards don't apply."""
        result = _read_csv_parallel(str(path), kwds, n_workers)
        if result is None:
            pytest.skip("parallel read not applicable to this file")
        return result

    def _base_kwds(self, path, **overrides):
        """Return kwds as _read() would see them for a default read_csv call."""
        from pandas._libs import lib

        from pandas.io.parsers.readers import _refine_defau
…[8629 chars omitted]…
/ "big.csv"
    _make_large_csv(path)
    # Lower the threshold so any file triggers the parallel path.
    monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)

    serial = read_csv(path, engine="python")
    # Force the parallel path so this runs on every platform (off by default on
    # Windows).
    with option_context("mode.max_threads", 4):
        parallel = read_csv(path, engine="c")

def test_ints_and_floats(self, tmp_path):
        path = tmp_path / "data.csv"
        n = 5_000
        rng = np.random.default_rng(0)
        df = DataFrame({"a": rng.integers(0, 100, n), "b": rng.random(n)})
        df.to_csv(path, index=False)
        kwds = self._base_kwds(path)
        result = self._parallel_read(path, kwds)
        expected = self._serial_read(path)
        tm.assert_frame_equal(result.reset_index(drop=True), expected)

    def test_strings(self, tmp_path):

        path = tmp_path / "data.csv"
        n = 5_000
        rng = np.random.default_rng(1)
        df = DataFrame(
            {
                "name": rng.choice(["alice", "bob", "carol"], n),
                "val": rng.integers(0, 10, n),
            }
        )
        df.to_csv(path, index=False)
        kwds = self._base_kwds(path)
        result = self._parallel_read(path, kwds)
        expect
…[8630 chars omitted]…
#64347).
    lines = ["a,b"] + ["\ufeff" + f"x{i},{i}" for i in range(4000)]
    path = tmp_path / "bom.csv"
    path.write_text("\n".join(lines) + "\n", encoding="utf-8")

    with option_context("mode.max_threads", 1):
        expected = read_csv(path)
    result = _read_forced_parallel(path, monkeypatch)
    tm.assert_frame_equal(result, expected)
    assert (result["a"].str[0] == "\ufeff").all()

pytest.param(
            b'a,b\n1,"line1\nline2\nline3"\n2,normal\n3,"also\nembedded"\n',
            3,
            id="multiple_embedded_newlines",
        ),
    ],
)
def test_embedded_newline_falls_back_to_serial(
    tmp_path, monkeypatch, csv_bytes, expected_rows
):
    """
    CSV files with newlines inside quoted fields must produce the same result

    as a direct serial read.  The parallel path either parses correctly (when
    the split misses the embedded newline) or falls back to serial (when it
    hits it).  Silent data corruption must never occur.
    """
    monkeypatch.setattr("pandas.io.parsers.readers._PARALLEL_READ_MIN_BYTES", 1)
    path = tmp_path / "embedded.csv"
    path.write_bytes(csv_bytes)

    serial = read_csv(io.BytesIO(csv_bytes))
    parallel = read_csv(path)  # may take parallel path or fall back

    assert len(parallel) == expected_rows
   
…[4665 chars omitted]…
sitting exactly at a chunk boundary: the chunk
    # worker's first line used to be exempt from the field-count check, so
    # parallel silently truncated the row while serial raised (GH#64347).
    path = tmp_path / "ragged.csv"
    _write_with_line_at_chunk_start(path, b"11,22,33,4444")

    with pytest.raises(ParserError, match="Expected 2 fields"):
        _read_forced_parallel(path, monkeypatch)

def _read_forced_parallel(path, monkeypatch, **kwargs):
    """read_csv with the parallel path force-enabled on every platform."""
    monkeypatch.setattr("pandas.io.parsers.readers._PARALLEL_READ_MIN_BYTES", 1)
    with option_context("mode.max_threads", 4):
        return read_csv(path, **kwargs)


def test_parallel_latin1_matches_serial(tmp_path, monkeypatch):
    # Non-UTF-8 encodings must take the serial path; the C tokenizer decodes
    # raw chunk bytes as UTF-8 (GH#64347).
    raw = b"col1,col2\n" + b"".join(
        f"se\xf1or{i},{i}\n".encode("latin-1") for i in range(500)

    )
    path = tmp_path / "latin.csv"
    path.write_bytes(raw)

    result = _read_forced_parallel(path, monkeypatch, encoding="latin-1")
    expected = read_csv(io.BytesIO(raw), encoding="latin-1")
    tm.assert_frame_equal(result, expected)
    assert result.loc[0, "col1"] == "se\xf1or0"


def test_pa
…[8112 chars omitted]…
0.csv"
    path.write_bytes(b"\xef\xbb\xbf" + body.encode())

    with option_context("mode.max_threads", 1):
        expected = read_csv(path, header=None, names=["a", "b"])
    result = _read_forced_parallel(path, monkeypatch, header=None, names=["a", "b"])
    tm.assert_frame_equal(result, expected)
    # the stripped BOM means the first cell parses as an integer
    assert result["a"].iloc[0] == 0

skip_blank_lines: bool
        parse_dates: bool | Sequence[Hashable] | None
        date_format: str | dict[Hashable, str] | None
        dayfirst: bool
        cache_dates: bool
        compression: CompressionOptions
        thousands: str | None
        decimal: str
        lineterminator: str | None
        quotechar: str
        quoting: int
        doublequote: bool

        escapechar: str | None
        comment: str | None
        encoding: str | None
        encoding_errors: str | None
        dialect: str | csv.Dialect | type[csv.Dialect] | None
        on_bad_lines: str
        low_memory: bool
        memory_map: bool
        float_precision: Literal["high", "legacy", "round_trip"] | None
        storage_options: StorageOptions | None
        dtype_backend: DtypeBackend | lib.NoDefault

else:
    _read_shared = dict


class _C_Parser_Defaults(TypedDict):
    na_fil
…[797 chars omitted]…
at_precision",
    "chunksize",
    "comment",
    "nrows",
    "thousands",
    "memory_map",
    "dialect",
    "quoting",
    "lineterminator",
    "converters",
    "iterator",
    "dayfirst",
    "skipinitialspace",
    "low_memory",
    "na_filter",
}


@overload
def validate_integer(name: str, val: None, min_val: int = ...) -> None: ...


@overload
0+untagged.1.g4622c59 /repo/pandas/__init__.py

URLError <urlopen error [Errno -3] Temporary failure in name resolution>

Refused: `find /` walks the whole container FS. Scope under /repo.

commit 4622c59d52248da9411aa62e6cf8a78c9e84cf95
Author: benchgen <sandbox@benchgen>
Date:   Fri Nov 6 20:44:45 2026 +0000

    base

 .gitattributes                                     |    82 +
 .github/CODEOWNERS                                 |    19 +
 .github/ISSUE_TEMPLATE/bug_report.yaml             |    71 +
 .../ISSUE_TEMPLATE/documentation_improvement.yaml  |    42 +
 .github/ISSUE_TEMPLATE/feature_request.yaml        |    73 +
 .github/ISSUE_TEMPLATE/installation_issue.yaml     |    66 +

 .github/ISSUE_TEMPLATE/pdep_vote.yaml              |    74 +
 .github/ISSUE_TEMPLATE/performance_issue.yaml      |    55 +
 .github/ISSUE_TEMPLATE/submit_question.yml         |    44 +
 .github/PULL_REQUEST_TEMPLATE.md                   |     7 +
 .github/actions/run-tests/action.yml               |    41 +
 .github/actions/setup-pixi/action.yml              |    18 +
 .github/dependabot.
…[8631 chars omitted]…
ow_memory",
408:    * The file is at least ``_PARALLEL_READ_MIN_BYTES`` bytes large.
534:        if file_size < _PARALLEL_READ_MIN_BYTES:
547:    if file_size - data_start < _PARALLEL_READ_MIN_BYTES:
891:    low_memory: bool = _c_parser_defaults["low_memory"],
1225:    low_memory : bool, default True
1485:    low_memory: bool = _c_parser_defaults["low_memory"],
1815:    low_memory : bool, default True

cp312
build/cp312/.gitignore
build/cp312/.hgignore
build/cp312/.ninja_deps
build/cp312/.ninja_log
build/cp312/build.ninja
build/cp312/compile_commands.json
build/cp312/meson-info/intro-benchmarks.json
build/cp312/meson-info/intro-buildoptions.json
build/cp312/meson-info/intro-buildsystem_files.json
build/cp312/meson-info/intro-compilers.json
build/cp312/meson-info/intro-dependencies.json

build/cp312/meson-info/intro-installed.json
build/cp312/meson-info/intro-install_plan.json
build/cp312/meson-info/intro-machines.json
build/cp312/meson-info/intro-projectinfo.json
build/cp312/meson-info/intro-targets.json
build/cp312/meson-info/intro-tests.json
build/cp312/meson-info/meson-info.json
build/cp312/meson-logs/meson-log.txt
build/cp312/meson-logs/meson-python-build-log.txt
build/cp312/meson-logs/meson-setup.txt
build/cp312/meson-private/build.dat
build/cp312/meson-private/cleantrees.dat
build/cp312/meson-private/cmd_line.txt
build/cp312/meson-private/coredata.dat
build/cp312/meson-private/install.dat
build/cp312/meson-private/meson.lock
build/cp312/meson-private/meson_benchmark_setup.dat
build/cp312/meson-private/meson_test_setup.dat
build/cp312/meson-private/pycompile.py

/root/.cache/pip/selfcheck/fe300af6f7d708c14827daac3afc81fbb8306b73de8dd6e3f1f8ea3b
/root/.cache/pip/http-v2/1/0/0/d/a/100da2520b2e40c82bcb1b7bb9cdfb6ab76bf0cc9e7b77c11667ba86
/root/.cache/pip/http-v2/1/0/0/d/a/100da2520b2e40c82bcb1b7bb9cdfb6ab76bf0cc9e7b77c11667ba86.body
/root/.cache/pip/http-v2/1/c/d/e/2/1cde2bb5bf17d6a419c193b554ece1ca0f901a08238c2c568e0647ed.body
/root/.cache/pip/http-v2/1/c/d/e/2/1cde2bb5bf17d6a419c193b554ece1ca0f901a08238c2c568e0647ed
/root/.cache/pip/http-v2/1/8/3/9/9/1839911afa3a298741e207e7e99aea7177030d328debd377548f19e2.body
/root/.cache/pip/http-v2/1/8/3/9/9/1839911afa3a298741e207e7e99aea7177030d328debd377548f19e2
/root/.cache/pip/http-v2/1/6/2/f/0/162f02b59107f7f8725a208a8de4172a22557991b6f4dc5cc182b640
/root/.cache/pip/http-v2/1/6/2/f/0/162f02b59107f7f8725a208a8de4172a22557991b6f4dc5cc182b640.body
/root/.cache/pip/http-v2/b/1/1/9/0/b11909f451f61fb2963621
…[3436 chars omitted]…

/root/.cache/pip/http-v2/5/1/0/c/2/510c26d886aa7ced2445589e570cd4d428b120de81d9a8975ff49aaf.body
/root/.cache/pip/http-v2/a/1/e/0/a/a1e0a86bad6bf000641a3b4ec8542efc4fb20564c3b5b964ae34d70f
/root/.cache/pip/http-v2/a/1/e/0/a/a1e0a86bad6bf000641a3b4ec8542efc4fb20564c3b5b964ae34d70f.body
/root/.cache/pip/http-v2/a/1/9/5/3/a19537d3cf37c122db841d6fe4cd322bc10d1a558bb00d146b85cb9a.body
30M	/root/.cache/pip

bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found

bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found

…[804 chars omitted]…
mand not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found
bash: line 58: file: command not found

zip 46785 /root/.cache/pip/http-v2/1/0/0/d/a/100da2520b2e40c82bcb1b7bb9cdfb6ab76bf0cc9e7b77c11667ba86.body
zip 129956 /root/.cache/pip/http-v2/b/1/1/9/0/b11909f451f61fb29636217082b1c638b177c59eac335e0200a136e6.body
gzip 951400 /root/.cache/pip/http-v2/b/c/9/7/8/bc978b74c647a5f4011ebd1fe3778a9c54d8a6dd43d688fbb4aa1959.body
zip 7552 /root/.cache/pip/http-v2/b/8/e/6/d/b8e6d5935f9ced55635d32bdf46f328b37f77b0474abe35d0a0dbc83.body
zip 1250147 /root/.cache/pip/http-v2/0/0/4/f/5/004f51e4cdaa64d7951a6294c47f410a2290cde6415bd52c394181fd.body
gzip 10411 /root/.cache/pip/http-v2/0/c/f/6/e/0cf6e817e2c5554000c735ecab0f3cf492f7d33b50d5a474a801ba24.body
gzip 47058 /root/.cache/pip/http-v2/0/c/2/1/8/0c218d526767991766d9b365a4f3e4449ce3579fd8b0f96a707d5a52.body
gzip 6928 /root/.cache/pip/http-v2/0/f/a/e/c/0faec10fc908d92f19ad98c40f513a1781ae2d868b0add6bff108aff.body
gzip 47155 /root/.cache/pip/http-v2
…[2580 chars omitted]…
p/http-v2/2/b/1/d/2/2b1d293c3d18b4323770d2cd4fd26b0d9830fd0996d25d67d5bf7d2c.body
gzip 5609 /root/.cache/pip/http-v2/2/4/0/c/3/240c3db8881a7d6c8664787e583e7d994c8e880437c45b37aa0bdd98.body

zip 411776 /root/.cache/pip/http-v2/2/5/6/0/3/25603daf65c6bff9032b0bd6188db919b28c8f70c749b7077d1e2521.body
gzip 22817 /root/.cache/pip/http-v2/7/a/5/f/7/7a5f717e7b3fb99ec006f4d173aad55224eb063005c662c586ba0518.body

/root/.cache/pip/http-v2/1/0/0/d/a/100da2520b2e40c82bcb1b7bb9cdfb6ab76bf0cc9e7b77c11667ba86.body 46785 ['versioneer.py', 'versioneer-0.29.dist-info/LICENSE', 'versioneer-0.29.dist-info/METADATA', 'versioneer-0.29.dist-info/WHEEL', 'versioneer-0.29.dist-info/entry_points.txt']
/root/.cache/pip/http-v2/b/1/1/9/0/b11909f451f61fb29636217082b1c638b177c59eac335e0200a136e6.body 129956 ['packaging/__init__.py', 'packaging/_elffile.py', 'packaging/_manylinux.py', 'packaging/_musllinux.py', 'packaging/_parser.py']
/root/.cache/pip/http-v2/b/8/e/6/d/b8e6d5935f9ced55635d32bdf46f328b37f77b0474abe35d0a0dbc83.body 7552 ['iniconfig/__init__.py', 'iniconfig/_parse.py', 'iniconfig/_version.py', 'iniconfig/exceptions.py', 'iniconfig/py.typed']
/root/.cache/pip/http-v2/0/0/4/f/5/004f51e4cdaa64d7951a6294c47f410a2290cde6415bd52c394181fd.body 1250147 ['pygments/__init__.py', 'pygments/__main__.py', 'pygment
…[3026 chars omitted]…
/1/d/2/2b1d293c3d18b4323770d2cd4fd26b0d9830fd0996d25d67d5bf7d2c.body 92853 ['dateutil/__init__.py', 'dateutil/_common.py', 'dateutil/_tzdata_impl.py', 'dateutil/_version.py', 'dateutil/easter.py']
/root/.cache/pip/http-v2/2/5/6/0/3/25603daf65c6bff9032b0bd6188db919b28c8f70c749b7077d1e2521.body 411776 ['py.py', '_pytest/__init__.py', '_pytest/_argcomplete.py', '_pytest/_version.py', '_pytest/approx.py']

# pandas Agent Instructions

## Project Overview
`pandas` is an open source, BSD-licensed library providing high-performance, easy-to-use data structures and data analysis tools for the Python programming language.

## Purpose
- Assist contributors by suggesting code changes, tests, and documentation edits for the pandas repository while preserving stability and compatibility.

## Persona & Tone
- Concise, neutral, code-focused. Prioritize correctness, readability, and tests.

## Project Guidelines

- Be sure to follow all guidelines for contributing to the codebase specified at https://pandas.pydata.org/docs/development/contributing_codebase.html
- These guidelines are also available in the following local files, which should be loaded into context and adhered to
    - doc/source/development/contributing_codebase.rst
    - doc/source/development/contributing_docstring.rst
    - doc/so
…[1525 chars omitted]…
  - CLN: Code cleanup
- Pull request descriptions should follow the template, and **succinctly** describe the change being made. Usually a few sentences is sufficient.
- Pull requests which are resolving an existing Github Issue should include a link to the issue in the PR Description.
- Do not add summaries or additional comments to individual commit messages. The single PR description is sufficient.

Parameter name (used for error reporting)
    val : int or float
        The value to check
    min_val : int
        Minimum allowed value (val < min_val will result in a ValueError)
    """
    if val is None:
        return val

    msg = f"'{name:s}' must be an integer >={min_val:d}"
    if is_float(val):
        if int(val) != val:

            raise ValueError(msg)
        val = int(val)
    elif not (is_integer(val) and val >= min_val):
        raise ValueError(msg)

    return int(val)


def _validate_names(names: Sequence[Hashable] | None) -> None:
    """
    Raise ValueError if the `names` parameter contains duplicates or has an
    invalid data type.

    Parameters
    ----------
    names : array-like or None
        An array containing a list of the names used for the output DataFrame.

    Raises
    ------
    ValueError
        If names are not unique or are n
…[4049 chars omitted]…
   # signal a parallel-path bug, not ineligible input.
                result = None
            if result is not None:
                return result

    # Create the parser.
    parser = TextFileReader(filepath_or_buffer, **kwds)

    if chunksize or iterator:
        return parser

    with parser:
        return parser.read(nrows)


def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:

800:def read_csv(
810:def read_csv(
820:def read_csv(
830:def read_csv(
840:def read_csv(
2806:def _refine_defaults_read(
    header: int | Sequence[int] | None | Literal["infer"] = "infer",
    names: Sequence[Hashable] | None | lib.NoDefault = lib.no_default,
    index_col: IndexLabel | Literal[False] | None = None,
    usecols: UsecolsArgType = None,
    # General Parsing Configuration
    dtype: DtypeArg | None = None,

    engine: CSVEngine | None = None,
    converters: Mapping[HashableT, Callable] | None = None,
    true_values: list | None = None,
    false_values: list | None = None,
    skipinitialspace: bool = False,
    skiprows: list[int] | int | Callable[[Hashable], bool] | None = None,
    skipfooter: int = 0,
    nrows: int | None = None,
    # NA and Missing Data Handling
    na_values: (
        Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | N
…[8521 chars omitted]…
', 'python', 'pyarrow', 'python-fwf'}, optional
        Parser engine to use. If not specified, defaults to ``'python'``.
    **kwds
        Any keyword argument accepted by :func:`read_csv`, :func:`read_table`,
        or :func:`read_fwf`.

    See Also
    --------
    read_csv : Read a comma-separated values (csv) file into a DataFrame.
    read_table : Read general delimited file into a DataFrame.

1390:    return _read(filepath_or_buffer, kwds)
1974:    return _read(filepath_or_buffer, kwds)
2129:    return _read(
    >>> pd.read_csv("data.csv", dtype={"Value": float})  # doctest: +SKIP
       Name  Value
    0   foo    1.0
    1   bar    2.0
    2  #baz    3.0

    True, False, and NA values, and thousands separators have defaults,
    but can be explicitly specified, too. Supply the values you would like
    as strings or lists of strings!

    >>> pd.read_csv("data.csv", na_values=["foo", "bar"])  # doctest: +SKIP
       Name  Value
    0   NaN      1
    1   NaN      2
    2  #baz      3

    Comment lines in the input file can be skipped using the `comment` argument.

    >>> pd.read_csv("data.csv", comment="#")  # doctest: +SKIP
      Name  Value
    0  foo      1
    1  bar      2

    By default, columns with dates will be read as ``object`` rather than  ``datetime``.


…[2322 chars omitted]…
fault,
    delimiter: str | None | lib.NoDefault = None,
    # Column and Index Locations and Names
    header: int | Sequence[int] | None | Literal["infer"] = "infer",
    names: Sequence[Hashable] | None | lib.NoDefault = lib.no_default,
    index_col: IndexLabel | Literal[False] | None = None,
    usecols: UsecolsArgType = None,
    # General Parsing Configuration
    dtype: DtypeArg | None = None,

_c_unsupported = {"skipfooter"}
_python_unsupported = {"low_memory", "float_precision"}

# Minimum file size (bytes) to attempt parallel CSV reading.
# Below this threshold the overhead of splitting and threading outweighs the benefit.
_PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
_pyarrow_unsupported = {
    "skipfooter",
    "float_precision",
    "chunksize",
    "comment",
    # nrows: we'd need to distribute the row budget across chunks - skip for now.

    if kwds.get("nrows") is not None:
        return False

    # The chunk splitter scans for raw \n bytes.  A custom lineterminator
    # (which the C engine does honour) would misalign chunk boundaries.
    lineterminator = kwds.get("lineterminator")
    if lineterminator is not None and lineterminator != "\n":
        return False

    # Chunk workers feed raw file bytes to the C tokenizer, which decodes
    # words as U
…[3499 chars omitted]…
          if not fd.readline():
                break
        return fd.tell()


def _find_chunk_byte_offsets(
    filepath: str,
    n_chunks: int,
    data_start: int,
) -> list[int]:
    """
    Compute byte offsets that partition the data portion of *filepath* into
    *n_chunks* approximately equal pieces aligned to newline boundaries.

    Returns a list of ``n + 1`` offsets where the byte range

patched
 pandas/io/parsers/readers.py | 68 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 64 insertions(+), 4 deletions(-)
diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 97b77f2..4f8e3e9 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -183,6 +183,11 @@ _python_unsupported = {"low_memory", "float_precision"}
 # Minimum file size (bytes) to attempt parallel CSV reading.
 # Below this threshold the overhead of splitting and threading outweighs the benefit.
 _PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
+

+# Minimum number of data rows per chunk.  Splitting a file into chunks smaller
+# than this makes the per-chunk parser setup and dtype-inference overhead
+# outweigh the benefit of parallel execution.
+_PARALLEL_MIN_CHUNK_ROWS = 2000
 _pyarrow_unsupported = {
     "skipfooter",
     "float_precision",

…[4137 chars omitted]…
rows = _estimate_row_count(filepath, data_start)
+    if estimated_rows < 2 * _PARALLEL_MIN_CHUNK_ROWS:
+        return None
+    n_chunks = min(n_workers * 3, estimated_rows // _PARALLEL_MIN_CHUNK_ROWS)
+    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
     n_chunks = len(offsets) - 1
     if n_chunks < 2:
         # e.g. a data section with no interior newlines (one giant line)

..............................................................................
============================= slowest 30 durations =============================
0.08s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_auto_parallel
0.05s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_parallel_vs_serial_large_file
0.04s setup    pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.03s call     pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.02s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_default_off_on_windows
0.02s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_ints_and_floats
0.02s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_default_thread_cap
0.01s call     pandas/tests/io
…[613 chars omitted]…
pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_skiprows_int

0.01s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_dict_dtype_duplicate_names_falls_back
0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_ragged_line_at_chunk_start_raises

(14 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 1.50s

low_memory=False eligible? False
low_memory=True eligible? True
unset eligible? True
wide estimated rows 272
wide parallel result is None? True
wide read matches serial (300, 500)
giant data size 387783
giant estimated rows 32315
Traceback (most recent call last):
  File "<stdin>", line 76, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 742, in _read_csv_parallel
    name_reader = TextFileReader(name_buf, **base_kwds)

                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2282, in __init__
    self.options, self.engine = self._clean_options(options, engine)
                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2469, in _clean_options
    names = list(names) if names is not None else names
            ^^^^^^^^^^^
TypeError: '_NoDefault' object is not iterable

giant data size 387783
giant estimated rows 32315
giant requested chunks [12]
Traceback (most recent call last):
  File "<stdin>", line 34, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 1450, in read_csv
    return _read(filepath_or_buffer, kwds)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 367, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2275, in __init__

    options = self._get_options_with_defaults(engine)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2343, in _get_options_with_defaults
    raise ValueError(
ValueError: The 'low_memory' option is not supported with the 'python' engine

giant data size 387783
giant estimated rows 32315
giant requested chunks [12]
giant result shape (30001, 2) (30001, 2)
giant matches serial
lowmem size 4700002
serial dtypes {'a': <StringDtype(storage='python', na_value=nan)>} nulls {'a': 20000}
eligible low_memory=False False
low_memory=False read matches serial; dtypes {'a': <StringDtype(storage='python', na_value=nan)>} nulls {'a': 20000}
direct parallel returned None

pandas/tests/io/parser/common/test_chunksize.py:242:    if parser.engine == "c" and parser.low_memory:
pandas/tests/io/parser/common/test_chunksize.py:263:                "Specify dtype option on import or set low_memory=False.",
pandas/tests/io/parser/common/test_chunksize.py:266:    if parser.engine == "c" and parser.low_memory:
pandas/tests/io/parser/test_parallel_read.py:475:            "low_memory": True,

patched guard

    The caller must ensure the file is eligible via :func:`_can_parallelize_csv`
    before calling this function.

    Returns ``None`` when parallel reading turns out not to be applicable
    after all (the data section cannot be split, the engine falls back to
    python, or per-chunk dtype inference disagrees in a way that would not
    match the serial result); the caller then reads serially.  Splitting is
    done at raw ``\\n`` boundaries, so a boundary inside a quoted field raises
    ``ParserError`` from the affected worker - the caller treats that as a
    serial-fallback signal too.

    """
    # low_memory=False requires whole-file dtype inference, which cannot be
    # reproduced by parsing chunks independently.
    if kwds.get("low_memory") is False:
        return None

    # Resolve the effective header value (mirrors TextFileReader.__init__).
    header
…[904 chars omitted]…
ks, data_start)
    n_chunks = len(offsets) - 1
    if n_chunks < 2:
        # e.g. a data section with no interior newlines (one giant line)
        return None

    # ------------------------------------------------------------------
    # Infer column names from the preamble + one data line (very fast).
    # ------------------------------------------------------------------
    base_kwds: dict = {

0.02s call     pandas/tests/io/parser/test_multi_thread.py::test_multi_thread_path_multipart_read_csv[c_high]
0.02s setup    pandas/tests/io/parser/test_compression.py::test_compression[c_low-gzip-True-None]
0.02s call     pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
=========================== short test summary info ============================
ERROR pandas/tests/io/parser/common/test_file_buffer_url.py::test_url[c_high]
ERROR pandas/tests/io/parser/common/test_file_buffer_url.py::test_url[c_low]
ERROR pandas/tests/io/parser/common/test_file_buffer_url.py::test_url[python]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[gzip-python-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[gzip-python-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[gzip-c-explicit]
ERROR p
…[1199 chars omitted]…

_network.py::test_compressed_urls[tar-python-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-c-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-c-infer]
ERROR pandas/tests/io/parser/test_network.py::test_url_encoding_csv
ERROR pandas/tests/io/parser/test_read_fwf.py::test_url_urlopen
3977 passed, 960 skipped, 52 xfailed, 25 errors in 34.24s

/usr/local/bin/python: No module named ruff

41:    _find_chunk_byte_offsets,
95:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
104:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
167:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
182:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
195:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
208:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
218:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
226:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
237:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
245:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
267:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)

280:        monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_
…[902 chars omitted]…
inux")
737:    assert len(calls) == 1  # parallel by default elsewhere
746:    monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)
747:    monkeypatch.setattr(_readers.sys, "platform", "linux")
749:    monkeypatch.setattr(_readers.os, "cpu_count", lambda: 16)
757:    monkeypatch.setattr(_readers, "_read_csv_parallel", stub)
1002:    offsets = _find_chunk_byte_offsets(str(path), 4, data_start)

pandas/io/parsers/readers.py | 73 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 69 insertions(+), 4 deletions(-)
diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 97b77f2..628e632 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -183,6 +183,11 @@ _python_unsupported = {"low_memory", "float_precision"}
 # Minimum file size (bytes) to attempt parallel CSV reading.
 # Below this threshold the overhead of splitting and threading outweighs the benefit.
 _PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
+
+# Minimum number of data rows per chunk.  Splitting a file into chunks smaller

+# than this makes the per-chunk parser setup and dtype-inference overhead
+# outweigh the benefit of parallel execution.
+_PARALLEL_MIN_CHUNK_ROWS = 2000
 _pyarrow_unsupported = {
     "skipfooter",
     "float_precision",
@@ -401,
…[4626 chars omitted]…
rows = _estimate_row_count(filepath, data_start)
+    if estimated_rows < 2 * _PARALLEL_MIN_CHUNK_ROWS:
+        return None
+    n_chunks = min(n_workers * 3, estimated_rows // _PARALLEL_MIN_CHUNK_ROWS)
+    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
     n_chunks = len(offsets) - 1
     if n_chunks < 2:
         # e.g. a data section with no interior newlines (one giant line)

/root/.cache/pip/http-v2/5/1/0/c/2/510c26d886aa7ced2445589e570cd4d428b120de81d9a8975ff49aaf.body
/root/.cache/pip/http-v2/f/5/9/3/c/f593ca234aede623e188b076317f25b644843c4cd1fa65d559c75bcf.body

/root/.cache/pip/http-v2/f/5/9/3/c/f593ca234aede623e188b076317f25b644843c4cd1fa65d559c75bcf.body 5621 b'Metadata-Version: 2.4\nName: hypothesis\nVersion: 6.170.15\nClassifier: Development Status :: 5 - Production/Stable\nClassifier: Framework :: Hypothesis\nClassifier: Framework :: Pytest\nClassifier: Intende'
1517
b" hypothesis-crosshair>=0.0.31 ; extra == 'all'\nRequires-Dist: lark>=0.10.1 ; extra == 'all'\nRequires-Dist: libcst>=0.3.16 ; extra == 'all'\nRequires-Dist: numpy>=1.23.2 ; extra == 'all'\nRequires-Dist: pandas>=1.5 ; extra == 'all'\nRequires-Dist: pytest>=4.6 ; extra == 'all'\nRequires-Dist: python-dateutil>=1.4 ; extra == 'all'\nRequires-Dist: pytz>=2014.1 ; extra == 'all'\nRequires-Dist: redis>=3.0.0 ; extra == 'all'\nRequires-Dist: rich>=9.0.0 ; extra == 'all'\nRequires-Dist: tzdata>=2026.5 ; (sys_platform == 'emscripten' and extra == 'all') or (sys_platform == 'win32
…[695 chars omitted]…
VanLD63zz0pEagEyfUHlHIumw/YG768ZDHtuq91vGkRDAXbsrA8=\xa6Server\xa8AmazonS3\xadLast-Modified\xbdFri, 06 Nov 2026 09:54:56 GMT\xa4ETag\xd9""b551857a00336d0575b242c7de7d5f27"\xb0x-amz-version-id\xd9 1MdePUi3H4fNDFoZbkhaL9VobJ5WsX.g\xacContent-Type\xb3binary/octet-stream\xbcx-amz-server-side-encryption\xa6AES256\xafFastly-Restarts\xa11\xadCache-Control\xd9$max-age=365000000, immutable, public\xadAccept-R'

"""
Module contains tools for processing files into DataFrames or other objects

GH#48849 provides a convenient way of deprecating keyword arguments
"""

from __future__ import annotations

import codecs
from collections import (
    abc,
    defaultdict,

)
from concurrent.futures import ThreadPoolExecutor
import contextlib
import csv
import io
import mmap
import os
import sys
from typing import (
    IO,
    TYPE_CHECKING,
    Any,
    Generic,
    Literal,
    Self,
    TypedDict,
    Unpack,
    cast,

148:        low_memory: bool
160:    low_memory: Literal[True]
167:    "low_memory": True,
181:_python_unsupported = {"low_memory", "float_precision"}
206:    "low_memory",
409:    * ``low_memory`` is not ``False`` - whole-file dtype inference cannot be
461:    # low_memory=False infers dtypes from the whole file, which cannot be
464:    if kwds.get("low_memory") is False:
702:    # low_memory=False requires whole-file dtype inference, which cannot be
704:    if kwds.get("low_memory") is False:
956:    low_memory: bool = _c_parser_defaults["low_memory"],
1290:    low_memory : bool, default True

1550:    low_memory: bool = _c_parser_defaults["low_memory"],
1880:    low_memory : bool, default True
878:    def read_low_memory(self, rows: int | None)-> list[dict[int, "ArrayLike"]]:

1290:    low_memory : bool, default True
1291-        Internally process the file in chunks, resulting in lower memory use
1292-        while parsing, but possibly mixed type inference. Because each chunk
1293-        is type-inferred independently, the same literal can be parsed as an
--
1547-    # Error Handling
1548-    on_bad_lines: str = "error",
1549-    # Internal
1550:    low_memory: bool = _c_parser_defaults["low_memory"],
1551-    memory_map: bool = False,
1552-    float_precision: Literal["high", "legacy", "round_trip"] | None = None,
1553-    storage_options: StorageOptions | None = None,

--
1877-
1878-            Callable for ``engine='pyarrow'``
1879-
1880:    low_memory : bool, default True
1881-        Internally process the file in chunks, resulting in lower memory use
1882-        while parsing, but possibly mixed type inference. Because each chunk
1883-        is type-inferred independently, the same literal can be parsed as an

0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_bom_bytes_mid_file_match_serial
0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_ragged_line_at_chunk_start_raises

(15 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 1.42s

size 110003 data_start 2
estimate 55000
requested [12]
offsets actual chunks 2
matches serial (30001, 1)

patched helper


def _find_chunk_byte_offsets(
    filepath: str,
    n_chunks: int,
    data_start: int,
) -> list[int]:
    """
    Compute byte offsets that partition the data portion of *filepath* into
    *n_chunks* approximately equal pieces aligned to newline boundaries.

    Returns a list of ``n + 1`` offsets where the byte range
    ``[offsets[i], offsets[i+1])`` defines chunk *i*.
    """
    file_size = os.path.getsize(filepath)
    data_size = file_size - data_start
    offsets = [data_start]

    if n_chunks <= 1 or data_size <= 0:
        offsets.append(file_size)
        return offsets

    with open(filepath, "rb") as fd:
        for i in range(1, n_chunks):
            # Distribute the bytes that remain over the chunks that remain
            # rather than using a fixed target.  A single oversized line can
            # otherwise contain several fixed targets, so eac
…[159 chars omitted]…
s[-1] + (file_size - offsets[-1]) // remaining_chunks
            if target >= file_size:
                break
            fd.seek(target)
            fd.readline()  # advance past the partial line at the split point
            pos = fd.tell()
            if pos >= file_size or pos <= offsets[-1]:
                break
            offsets.append(pos)

    offsets.append(file_size)
    return offsets

requested [12]
actual chunks 12
matches serial (30001, 1)

3999 3777
4000 3778
4001 3779
5000 4778
10000 9778

3999 median 3777 mean 3924 [8, 8, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]
4000 median 3778 mean 3925 [8, 8, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]
4001 median 3779 mean 3926 [8, 8, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]
5000 median 4778 mean 4900 [8, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]
10000 median 9778 mean 9901 [8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10]
30000 median 28148 mean 29372 [10, 10, 10, 10, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12]

300 279
3999 3905
4000 3905
4001 3905
5000 4895
10000 9854
30000 29558
giant tiny 30937
giant var 29881
wide 146

params 16 4096 [279, 3905, 3905, 3905, 4895, 9854, 29558]
params 32 4096 [279, 3915, 3916, 3917, 4910, 9890, 29710]
params 64 4096 [279, 3922, 3923, 3924, 4920, 9906, 29820]
params 16 1024 [280, 3955, 3954, 3955, 4928, 9930, 29584]
params 32 1024 [281, 3950, 3951, 3955, 4947, 9930, 29646]
params 64 1024 [281, 3956, 3956, 3958, 4958, 9941, 29899]
params 32 256 [292, 3946, 3943, 3957, 4963, 9930, 29399]
params 64 256 [292, 3972, 3970, 3969, 4969, 9972, 29955]

300 [290, 290, 293, 292, 292]
3999 [3924, 3924, 3936, 3943, 3946]
4000 [3925, 3925, 3937, 3944, 3947]
4001 [3926, 3926, 3939, 3945, 3948]
5000 [4900, 4932, 4932, 4940, 4948]
10000 [9901, 9901, 9933, 9933, 9941]
30000 [29372, 29532, 29613, 29654, 29674]
giant [36931, 36931, 37041, 37152, 37180]

ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[xz-c-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-python-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-python-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-c-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-c-infer]
ERROR pandas/tests/io/parser/test_network.py::test_url_encoding_csv
ERROR pandas/tests/io/parser/test_read_fwf.py::test_url_urlopen
3977 passed, 960 skipped, 52 xfailed, 25 errors in 39.05s

infer_nrows: Literal[100]
    widths: None


_fwf_defaults: _Fwf_Defaults = {"colspecs": "infer", "infer_nrows": 100, "widths": None}
_c_unsupported = {"skipfooter"}
_python_unsupported = {"low_memory", "float_precision"}

# Minimum file size (bytes) to attempt parallel CSV reading.
# Below this threshold the overhead of splitting and threading outweighs the benefit.
_PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB

# Minimum number of data rows per chunk.  Splitting a file into chunks smaller
# than this makes the per-chunk parser setup and dtype-inference overhead
# outweigh the benefit of parallel execution.
_PARALLEL_MIN_CHUNK_ROWS = 2000
_pyarrow_unsupported = {
    "skipfooter",
    "float_precision",
    "chunksize",
    "comment",
    "nrows",
    "thousands",
    "memory_map",
    "dialect",
    "quoting",
    "lineterminator",
    "converters",
    "iterator",
    "d
…[8629 chars omitted]…
    n_chunks = len(offsets) - 1
    if n_chunks < 2:
        # e.g. a data section with no interior newlines (one giant line)
        return None

    # ------------------------------------------------------------------
    # Infer column names from the preamble + one data line (very fast).
    # ------------------------------------------------------------------
    base_kwds: dict = {
        **kwds,

M pandas/io/parsers/readers.py

[4, 152, 302, 446, 584]

fuzz ok

pandas/io/parsers/readers.py | 83 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 76 insertions(+), 7 deletions(-)
 M pandas/io/parsers/readers.py

eligibility ok
wide ok (300, 500)
giant ok (30001, 2) requested [12]
low_memory=False ok {'a': <StringDtype(storage='python', na_value=nan)>} {'a': 25000}

…[580 chars omitted]…
questing 12 chunks and matching serial). The existing parallel-read tests (78 passed) and full parser suite (3977 passed; only unrelated network tests errored) pass."}
submitted.

/repo
AGENTS.md
AUTHORS.md
CITATION.cff
LICENSE
LICENSES
MANIFEST.in
README.md
asv_bench
build
ci
codecov.yml

doc
environment.yml
generate_pxi.py
generate_version.py
meson.build
pandas
pixi.lock
pixi.toml
pyproject.toml
pyright_reportGeneralTypeIssues.json
requirements-dev.txt
scripts
subprojects
typings
web

1c9824d base

"""
Module contains tools for processing files into DataFrames or other objects

GH#48849 provides a convenient way of deprecating keyword arguments
"""

from __future__ import annotations

import codecs
from collections import (
    abc,
    defaultdict,

)
from concurrent.futures import ThreadPoolExecutor
import contextlib
import csv
import io
import mmap
import os
import sys
from typing import (
    IO,
    TYPE_CHECKING,
    Any,
    Generic,
    Literal,
    Self,
    TypedDict,
    Unpack,
    cast,
    overload,
)
import warnings

import numpy as np

from pandas._config import get_option

from pandas._libs import lib
from pandas._libs.parsers import STR_NA_VALUES
from pandas.errors import (
    AbstractMethodError,
    Pandas4Warning,
    ParserError,
    ParserWarning,
)
from pandas.util._decorators import (
    set_module,
)
from pandas.util._exceptions import find_stack_level
…[8630 chars omitted]…
)
    tm.assert_frame_equal(result, expected)
    assert result.loc[0, "col1"] == "se\xf1or0"


def test_parallel_ascii_non_ascii_byte_matches_serial(tmp_path, monkeypatch):
    # encoding="ascii" with a non-ASCII (but valid UTF-8) byte deep in the
    # file: the serial path raises UnicodeDecodeError, but the workers decode
    # chunk bytes as UTF-8 and would silently succeed, so ascii must take the

quoting: int
        doublequote: bool
        escapechar: str | None
        comment: str | None
        encoding: str | None
        encoding_errors: str | None
        dialect: str | csv.Dialect | type[csv.Dialect] | None
        on_bad_lines: str
        low_memory: bool
        memory_map: bool
        float_precision: Literal["high", "legacy", "round_trip"] | None
        storage_options: StorageOptions | None

        dtype_backend: DtypeBackend | lib.NoDefault

else:
    _read_shared = dict


class _C_Parser_Defaults(TypedDict):
    na_filter: Literal[True]
    low_memory: Literal[True]
    memory_map: Literal[False]
    float_precision: None


_c_parser_defaults: _C_Parser_Defaults = {
    "na_filter": True,
    "low_memory": True,
    "memory_map": False,
    "float_precision": None,
}


class _Fwf_Defaults(TypedDict):
    colspecs: Literal["infer"]
    infer_nrows: L
…[8629 chars omitted]…
 line
    # count disagrees with its logical row count (e.g. a quoted embedded
    # newline in the header), so data_start is wrong.
    n_first_rows = len(name_reader.read())
    name_reader.close()
    if n_first_rows != 1:
        return None

    # A dict ``dtype`` is applied per raw header name: when a name is repeated,
    # the serial path assigns that dtype to every de-duplicated column (``a``

try:
                result = _read_csv_parallel(_filepath, kwds, _n_workers)
            except (ParserError, UnicodeDecodeError):
                # e.g. a chunk boundary landed inside a quoted field containing
                # an embedded newline.  The serial path below handles anything
                # the parallel path cannot.  Other exceptions propagate: they
                # signal a parallel-path bug, not ineligible input.
                result = None
            if result is not None:
                return result

    # Create the parser.

    parser = TextFileReader(filepath_or_buffer, **kwds)

    if chunksize or iterator:
        return parser

    with parser:
        return parser.read(nrows)


def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
    """
    Return True when a ``read_csv`` call is eligible for parallel execution.

    Parallel
…[8629 chars omitted]…
r):
        # TextFileReader fell back to the python engine for a reason the
        # eligibility checks did not anticipate; load_buffer needs the C engine.
        name_reader.close()
        return None
    if name_reader._engine._reader.leading_cols:
        # Data rows have more fields than the header (implicit index).  The
        # chunk workers would fail on the extra field; bail out up front.

compression = kwds.get("compression", "infer")
    if isinstance(compression, dict):
        compression = compression.get("method")
    if compression is not None:
        try:
            if infer_compression(filepath, compression) is not None:
                return False
        except ValueError:
            # unrecognized compression argument; let the serial path raise
            return False

    # Iterator / chunked mode: caller handles chunking themselves.

    if kwds.get("iterator", False) or kwds.get("chunksize") is not None:
        return False

    # Only the C engine benefits; pyarrow has its own parallelism.
    # engine=None means "use the default", which is "c".
    if kwds.get("engine") not in ("c", None):
        return False

    # nrows: we'd need to distribute the row budget across chunks - skip for now.
    if kwds.get("nrows") is not None:
        return 
…[8629 chars omitted]…
["low_memory"],
    memory_map: bool = False,
    float_precision: Literal["high", "legacy", "round_trip"] | None = None,
    storage_options: StorageOptions | None = None,
    dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
) -> DataFrame | TextFileReader:
    """
    Read a comma-separated values (csv) file into DataFrame.

    Also supports optionally iterating or breaking of the file

if isinstance(kwds.get("header", "infer"), (list, tuple)):
        return False

    # skipfooter: the C engine doesn't support it anyway, but be explicit.
    if kwds.get("skipfooter", 0) > 0:
        return False

    # A column-based index_col would need its name propagated to non-first chunks.
    index_col = kwds.get("index_col", None)
    if index_col is not None and index_col is not False:
        return False

    # usecols changes the column name ↔ position mapping in non-first chunks.
    if kwds.get("usecols") is not None:
        return False

    # Only bother for files large enough to amortise the threading overhead.
    try:
        file_size = os.path.getsize(filepath)
        if file_size < _PARALLEL_READ_MIN_BYTES:
            return False
    except OSError:
        return False

    # Check that the *data section* (after header/skiprows preamble) is large
    #
…[4949 chars omitted]…
b") as fd:
        preamble = fd.read(data_start)
        first_line = fd.readline()

    name_buf = io.BytesIO(preamble + first_line)
    name_reader = TextFileReader(name_buf, **base_kwds)
    if not isinstance(name_reader._engine, CParserWrapper):
        # TextFileReader fell back to the python engine for a reason the
        # eligibility checks did not anticipate; load_buffer needs the C engine.

# eligibility checks did not anticipate; load_buffer needs the C engine.
        name_reader.close()
        return None
    if name_reader._engine._reader.leading_cols:
        # Data rows have more fields than the header (implicit index).  The
        # chunk workers would fail on the extra field; bail out up front.
        name_reader.close()
        return None
    assert name_reader._engine.orig_names is not None
    col_names: list = list(name_reader._engine.orig_names)
    # name_buf holds the preamble plus exactly one data line.  Parsing it must
    # yield exactly one row; anything else means the preamble's physical line

    # count disagrees with its logical row count (e.g. a quoted embedded
    # newline in the header), so data_start is wrong.
    n_first_rows = len(name_reader.read())
    name_reader.close()
    if n_first_rows != 1:
        return None

    # A di
…[4386 chars omitted]…
 FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
    *,
    iterator: Literal[False] = ...,
    chunksize: None = ...,
    **kwds: Unpack[_read_shared[HashableT]],
) -> DataFrame: ...


@overload
def read_csv(
    filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
    *,
    iterator: bool = ...,
    chunksize: int | None = ...,
    **kwds: Unpack[_read_shared[HashableT]],

expected = read_csv(io.BytesIO(raw))
    tm.assert_frame_equal(result, expected)
    assert len(result) == 500


def test_parallel_storage_options_raises(tmp_path, monkeypatch):
    # storage_options with a local path raises ValueError in the serial path;
    # the parallel path strips it and must not mask that error (GH#64347).
    raw = b"a,b\n" + b"".join(f"{i},{i}\n".encode() for i in range(500))
    path = tmp_path / "so.csv"
    path.write_bytes(raw)

    msg = "storage_options passed with file object or non-fsspec file path"
    with pytest.raises(ValueError, match=msg):
        _read_forced_parallel(path, monkeypatch, storage_options={"anon": True})


def test_parallel_on_bad_lines_warn_line_numbers(tmp_path, monkeypatch):
    # on_bad_lines="warn" must report absolute line numbers; chunk workers
    # would report chunk-relative ones, so the serial path is used (GH#64347)
…[5207 chars omitted]…
eff" + f"x{i},{i}" for i in range(4000)]
    path = tmp_path / "bom.csv"
    path.write_text("\n".join(lines) + "\n", encoding="utf-8")

    with option_context("mode.max_threads", 1):
        expected = read_csv(path)
    result = _read_forced_parallel(path, monkeypatch)
    tm.assert_frame_equal(result, expected)
    assert (result["a"].str[0] == "\ufeff").all()


475:            "low_memory": True,

path.write_bytes(b"r0\nr1\nr2_header\n1,2\n")
        offset = _find_data_start_offset(str(path), header=2, skiprows=0)
        assert offset == len(b"r0\nr1\nr2_header\n")

    def test_eof_within_preamble(self, tmp_path):
        # File ends before all preamble lines are consumed → no crash.
        path = tmp_path / "data.csv"
        path.write_bytes(b"only_one_line\n")
        offset = _find_data_start_offset(str(path), header=0, skiprows=5)
        assert offset == len(b"only_one_line\n")

# ---------------------------------------------------------------------------
# _find_chunk_byte_offsets
# ---------------------------------------------------------------------------


class TestFindChunkByteOffsets:
    def _write_csv(self, path: Path, n_rows: int = 10) -> None:
        lines = "a,b\n" + "".join(f"{i},{i}\n" for i in range(n_rows))
        path.write_text(lines, enco
…[8628 chars omitted]…
-------------------------------------------------------
# Embedded newlines in quoted fields
# ---------------------------------------------------------------------------


@pytest.mark.parametrize(
    "csv_bytes,expected_rows",
    [
        # Simple embedded newline
        pytest.param(
            b'a,b,c\n1,"hello\nworld",3\n4,normal,6\n',
            2,
            id="simple_embedded_newline",

89:    def test_eligible_large_file(self, tmp_path, monkeypatch):
98:    def test_accepts_default_engine(self, tmp_path, monkeypatch):
107:    def test_rejects_file_like_object(self, tmp_path):
113:    def test_rejects_url(self):
117:    def test_rejects_compressed(self, tmp_path):
123:    def test_rejects_tar(self, tmp_path):
130:    def test_rejects_dict_compression(self, tmp_path):
136:    def test_rejects_iterator_mode(self, tmp_path):
141:    def test_rejects_chunksize(self, tmp_path):
146:    def test_rejects_nrows(self, tmp_path):
151:    def test_rejects_non_c_engine(self, tmp_path):
157:    def test_rejects_custom_lineterminator(self, tmp_path, monkeypatch):

174:    def test_rejects_utf16_32_encoding(self, tmp_path, monkeypatch, encoding):
186:    def test_rejects_non_utf8_encoding(self, tmp_path, monkeypatch, encoding):
201:    def test_rejects_python_engine_seps(self, tmp_p
…[4965 chars omitted]…
ath, monkeypatch):
1023:def test_parallel_ragged_line_at_chunk_start_skip_matches_serial(tmp_path, monkeypatch):
1036:def test_parallel_bom_bytes_mid_file_match_serial(tmp_path, monkeypatch):
1040:    lines = ["a,b"] + ["\ufeff" + f"x{i},{i}" for i in range(4000)]
1051:def test_parallel_bom_at_file_start_header_none(tmp_path, monkeypatch):
1054:    body = "".join(f"{i},{i * 2}\n" for i in range(4000))

@overload
def validate_integer(name: str, val: float, min_val: int = ...) -> int: ...


@overload
def validate_integer(name: str, val: int | None, min_val: int = ...) -> int | None: ...


def validate_integer(
    name: str, val: int | float | None, min_val: int = 0
) -> int | None:
    """

    Checks whether the 'name' parameter for parsing is either
    an integer OR float that can SAFELY be cast to an integer
    without losing accuracy. Raises a ValueError if that is
    not the case.

    Parameters
    ----------
    name : str
        Parameter name (used for error reporting)
    val : int or float
        The value to check
    min_val : int
        Minimum allowed value (val < min_val will result in a ValueError)
    """
    if val is None:
        return val

    msg = f"'{name:s}' must be an integer >={min_val:d}"
    if is_float(val):
        if int(val) != val:
          
…[8629 chars omitted]…
        offset = _find_data_start_offset(str(path), header=0, skiprows=2)
        # 'skip1\n' + 'skip2\n' + 'a,b\n' = 6 + 6 + 4 = 16
        assert offset == len(b"skip1\nskip2\na,b\n")

    def test_header_2(self, tmp_path):
        # header=2 means lines 0,1,2 are consumed (lines 0-1 skipped, line 2 is header)
        path = tmp_path / "data.csv"
        path.write_bytes(b"r0\nr1\nr2_header\n1,2\n")

0+untagged.1.g1c9824d
/repo/pandas/__init__.py

eligible False
Traceback (most recent call last):
  File "<stdin>", line 17, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 682, in _read_csv_parallel
    name_reader = TextFileReader(name_buf, **base_kwds)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2215, in __init__
    options = self._get_options_with_defaults(engine)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2283, in _get_options_with_defaults
    raise ValueError(
ValueError: The 'low_memory' option is not supported with the 'python' engine

148:        low_memory: bool
160:    low_memory: Literal[True]
167:    "low_memory": True,
181:_python_unsupported = {"low_memory", "float_precision"}
201:    "low_memory",
891:    low_memory: bool = _c_parser_defaults["low_memory"],
1225:    low_memory : bool, default True
1485:    low_memory: bool = _c_parser_defaults["low_memory"],
1815:    low_memory : bool, default True
2190:            engine_specified = True
2193:            engine_specified = False
2195:        self._engine_specified = kwds.get("engine_specified", engine_specified)

2366:        if fallback_reason and self._engine_specified:
2891:        kwds["engine_specified"] = True
2894:        kwds["engine_specified"] = False
@overload
def read_fwf(
    filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
    *,
    colspecs: Sequence[tuple[int, int]] | str | None = ...,
    widths: Sequence[int] | Non
…[5244 chars omitted]…
 ValueError("Length of colspecs must match length of names")

    check_dtype_backend(kwds.setdefault("dtype_backend", lib.no_default))
    return _read(
        filepath_or_buffer,
        kwds
        | {
            "colspecs": colspecs,
            "infer_nrows": infer_nrows,
            "engine": "python-fwf",
            "iterator": iterator,
            "chunksize": chunksize,
        },
    )

bytes 170006
eligible True
len 15000 15000
dtypes value    str
dtype: object value    str
dtype: object
serial ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, nan, nan, nan, nan]
parallel ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, nan, nan, nan, nan]
na counts [5000] [5000]
equal True

# Same as the comment case, but triggered by a plain blank line (GH#64347).
    raw = b"\ncol1,col2\n" + b"".join(f"{i},{i * 2}\n".encode() for i in range(500))
    path = tmp_path / "blank.csv"
    path.write_bytes(raw)

    result = _read_forced_parallel(path, monkeypatch)
    expected = read_csv(io.BytesIO(raw))
    tm.assert_frame_equal(result, expected)
    assert result["col1"].dtype == np.int64


def test_parallel_single_long_line(tmp_path, monkeypatch):

    # A data section with no interior newlines cannot be split; this must
    # fall back to serial instead of raising (GH#64347).
    raw = b"col1,col2\n" + b"a" * 5_000 + b",b"
    path = tmp_path / "longline.csv"
    path.write_bytes(raw)

    result = _read_forced_parallel(path, monkeypatch)
    expected = read_csv(io.BytesIO(raw))
    tm.assert_frame_equal(result, expected)


def test_parallel_mixed_dtype_column_matche
…[3305 chars omitted]…
one_line\n")
        offset = _find_data_start_offset(str(path), header=0, skiprows=5)
        assert offset == len(b"only_one_line\n")

210:[tool.ruff]
215:[tool.ruff.lint]
408:[tool.ruff.lint.flake8-tidy-imports.banned-api]
422:[tool.ruff.lint.flake8-import-conventions.aliases]
425:[tool.ruff.lint.per-file-ignores]
433:[tool.ruff.lint.flake8-pytest-style]
437:[tool.ruff.format]
658:profile = "black"

Traceback (most recent call last):
  File "<stdin>", line 28, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 682, in _read_csv_parallel
    name_reader = TextFileReader(name_buf, **base_kwds)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2222, in __init__
    self.options, self.engine = self._clean_options(options, engine)
                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2409, in _clean_options
    names = list(names) if names is not None else names
            ^^^^^^^^^^^
TypeError: '_NoDefault' object is not iterable

"skip_blank_lines": True,
            "parse_dates": None,
            "date_format": None,
            "dayfirst": False,
            "cache_dates": True,
            "iterator": False,
            "chunksize": None,
            "compression": "infer",
            "thousands": None,
            "decimal": ".",
            "lineterminator": None,
            "quotechar": '"',

            "quoting": 0,
            "doublequote": True,
            "escapechar": None,
            "comment": None,
            "encoding": None,
            "encoding_errors": "strict",
            "dialect": None,
            "on_bad_lines": "error",
            "low_memory": True,
            "memory_map": False,
            "float_precision": None,
            "storage_options": None,
            "dtype_backend": lib.no_default,
        }
        kwds.update(overrides)
        # Apply the same
…[821 chars omitted]…
e({"a": rng.integers(0, 100, n), "b": rng.random(n)})
        df.to_csv(path, index=False)
847:    names: Sequence[Hashable] | None | lib.NoDefault = lib.no_default,
1441:    names: Sequence[Hashable] | None | lib.NoDefault = lib.no_default,
2109:    if names is not None and names is not lib.no_default:
2806:def _refine_defaults_read(
2871:    kwds["names"] = None if names is lib.no_default else names

# Thus, we need a flag to indicate that we need to "override"
    # the comparison to dialect values by checking if default values
    # for BOTH "delimiter" and "sep" were provided.
    if dialect is not None:
        kwds["sep_override"] = delimiter is None and (
            sep is lib.no_default or sep == delim_default
        )

    if delimiter and (sep is not lib.no_default):
        raise ValueError("Specified a sep and a delimiter; you can only specify one.")

    kwds["names"] = None if names is lib.no_default else names

    # Alias sep -> delimiter.
    if delimiter is None:
        delimiter = sep

    if delimiter == "\n":
        raise ValueError(
            r"Specified \n as separator or delimiter. This forces the python engine "
            "which does not accept a line terminator. Hence it is not allowed to use "
            "the line terminator as separator.",
 
…[609 chars omitted]…
e.BadLineHandleMethod.BLHM_SKIP
    elif callable(on_bad_lines):
        if engine not in ["python", "pyarrow"]:
            raise ValueError(
                "on_bad_line can only be a callable function "
                "if engine='python' or 'pyarrow'"
            )
        kwds["on_bad_lines"] = on_bad_lines
    else:
        raise ValueError(f"Argument {on_bad_lines} is invalid for on_bad_lines")

eligible after? True
dtypes value    str
dtype: object value    str
dtype: object
na counts [5000] [5000]
first around ['ordinary', 'ordinary', nan, nan, nan, nan, nan, nan]
eq True

called kwargs False None c 4
parallel helper True None None
result value    str
dtype: object 5000

offsets 5 [0.0, 0.25, 0.5, 0.75, 1.0]
0 7372 str ['ordinary0', 'ordinary1', 'ordinary2'] ['ordinary7370', 'ordinary7371'] 0
1 13607 str ['ordinary7372', 'ordinary7373', 'ordinary7374'] ['18446744073709552493', '18446744073709552494'] 10000
2 4511 object [18446744073709552495, 18446744073709552496, 18446744073709552497] [18446744073709557004, 18446744073709557005] 0
3 4510 object [18446744073709557006, 18446744073709557007, 18446744073709557008] [18446744073709561514, 18446744073709561515] 0
full value    str
dtype: object 10000 ['ordinary9997', 'ordinary9998', 'ordinary9999', nan, nan, nan]
Traceback (most recent call last):
  File "<stdin>", line 26, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 682, in _read_csv_parallel
    name_reader = TextFileReader(name_buf, **base_kwds)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/repo/pandas/io/parsers/read
…[128 chars omitted]…
^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2504, in _make_engine
    return mapping[engine](f, **self.options)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/c_parser_wrapper.py", line 79, in __init__
    kwds["on_bad_lines"] = self.on_bad_lines.value
                           ^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'str' object has no attribute 'value'

5000 500 None
5000 5000 None
5000 10000 None
5000 30000 None
10000 500 None
10000 5000 None
10000 10000 None
10000 30000 None
30000 500 None
30000 5000 None
30000 10000 None
30000 30000 None

100000 500 None
100000 5000 None
100000 10000 None
100000 30000 None

Traceback (most recent call last):
  File "<stdin>", line 25, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 2228, in __init__
    self._engine = self._make_engine(f, self.engine)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2504, in _make_engine
    return mapping[engine](f, **self.options)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/c_parser_wrapper.py", line 95, in __init__
    self._reader = parsers.TextReader(src, **kwds)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "pandas/_libs/parsers.pyx", line 429, in pandas._libs.parsers.TextReader.__cinit__

    if len(delimiter) > 1:
TypeError: object of type 'NoneType' has no len()

def _base_kwds(self, path, **overrides):
        """Return kwds as _read() would see them for a default read_csv call."""
        from pandas._libs import lib

        from pandas.io.parsers.readers import _refine_defaults_read

        kwds = {
            "header": "infer",
            "names": lib.no_default,
            "index_col": None,
            "usecols": None,
            "dtype": None,

            "converters": None,
            "true_values": None,
            "false_values": None,
            "skipinitialspace": False,
            "skiprows": None,
            "skipfooter": 0,
            "nrows": None,
            "na_values": None,
            "keep_default_na": True,
            "na_filter": True,
            "skip_blank_lines": True,
            "parse_dates": None,
            "date_format": None,
            "dayfirst": False,
            "cache_dates": True,
 
…[817 chars omitted]…
faults_read(
            dialect=None,
            delimiter=lib.no_default,
            engine=None,  # triggers engine="c"
            sep=lib.no_default,
            on_bad_lines=kwds.pop("on_bad_lines"),
            names=kwds.pop("names"),
            defaults={"delimiter": ","},
            dtype_backend=kwds.pop("dtype_backend"),
        )
        kwds.update(kwds_defaults)
        return kwds

15 str 5 0 [(<StringDtype(storage='python', na_value=nan)>, np.int64(0), np.int64(0)), (dtype('float64'), np.int64(5), np.int64(0)), (dtype('uint64'), np.int64(0), np.int64(0))]
15 str 5 0 [(<StringDtype(storage='python', na_value=nan)>, np.int64(0), np.int64(0)), (dtype('float64'), np.int64(5), np.int64(0)), (dtype('uint64'), np.int64(0), np.int64(0))]
20 str 5 0 [(<StringDtype(storage='python', na_value=nan)>, np.int64(1), np.int64(0)), (<StringDtype(storage='python', na_value=nan)>, np.int64(0), np.int64(4)), (dtype('O'), np.int64(0), np.int64(0)), (dtype('O'), np.int64(0), np.int64(0))]
15000 str 5000 0 [(<StringDtype(storage='python', na_value=nan)>, np.int64(0), np.int64(0)), (dtype('float64'), np.int64(5000), np.int64(0)), (dtype('O'), np.int64(0), np.int64(0))]

pattern 3
True str 1 0 ['ordinary', nan, '18446744073709551615']
False str 1 0 ['ordinary', nan, '18446744073709551615']
pattern 15
True str 5 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, nan, nan]
False str 5 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, nan, nan]
pattern 11
True str 5 0 ['ordinary', nan, nan, nan, nan, nan, '18446744073709551615', '18446744073709551615']
False str 5 0 ['ordinary', nan, nan, nan, nan, nan, '18446744073709551615', '18446744073709551615']
pattern 11
True str 1 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, '18446744073709551615', '18446744073709551615']
False str 1 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, '18446744073709551615', '18446744073709551615']

pattern 11
True str 5 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, nan, nan]
False str 5 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', nan, nan, nan]
pattern 300
True str 100 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary']
False str 100 0 ['ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary', 'ordinary']

File "<stdin>", line 33
    else: print('no mismatch, successfuls?')
    ^^^^
SyntaxError: invalid syntax

File "<stdin>", line 32
SyntaxError: 'break' outside loop

scanned 50000 success 1
scanned 100000 success 8
successful 17 mismatch False

File "<stdin>", line 27
    else: continue
    ^^^^
SyntaxError: invalid syntax

MISMATCH 5000 5000 188892 str str NA 1223 5000 literal nan 3777 0 different rows 3777
res slice ['ordinary4998', 'ordinary4999', nan, '18446744073709551615', nan, '18446744073709551614', nan]
found True

eligible? True
equals True na 5000 5000 literal 0 0 types str str

low_memory=False eligible: True
mismatched rows: 3777
literal nan in parallel: 3777
300-row wide file returns serial fallback: False
large-first requested chunks: [4, 4]

pytest.param(
            b'a,b\n1,"line1\nline2\nline3"\n2,normal\n3,"also\nembedded"\n',
            3,
            id="multiple_embedded_newlines",
        ),
    ],
)
def test_embedded_newline_falls_back_to_serial(
    tmp_path, monkeypatch, csv_bytes, expected_rows
):
    """
    CSV files with newlines inside quoted fields must produce the same result

    as a direct serial read.  The parallel path either parses correctly (when
    the split misses the embedded newline) or falls back to serial (when it
    hits it).  Silent data corruption must never occur.
    """
    monkeypatch.setattr("pandas.io.parsers.readers._PARALLEL_READ_MIN_BYTES", 1)
    path = tmp_path / "embedded.csv"
    path.write_bytes(csv_bytes)

    serial = read_csv(io.BytesIO(csv_bytes))
    parallel = read_csv(path)  # may take parallel path or fall back

    assert len(parallel) == expected_rows
   
…[6066 chars omitted]…
hat are not eligible fall back to the serial path, and the result is
always identical to a serial read.

The number of threads is controlled with the ``mode.max_threads`` option,
which defaults to the number of CPU cores, capped at ``4``. On Windows the
default is ``1`` (serial), as parallel reading currently does not improve
performance there. Set the option to ``1`` to disable parallel reading, e.g.

infer_nrows: Literal[100]
    widths: None


_fwf_defaults: _Fwf_Defaults = {"colspecs": "infer", "infer_nrows": 100, "widths": None}
_c_unsupported = {"skipfooter"}
_python_unsupported = {"low_memory", "float_precision"}

# Minimum file size (bytes) to attempt parallel CSV reading.
# Below this threshold the overhead of splitting and threading outweighs the benefit.
_PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
_PARALLEL_MIN_CHUNK_ROWS = 2_000

_pyarrow_unsupported = {
    "skipfooter",
    "float_precision",
    "chunksize",
    "comment",
    "nrows",
    "thousands",
    "memory_map",
    "dialect",


def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
    """
    Return True when a ``read_csv`` call is eligible for parallel execution.

    Parallel reading works by splitting the file's data section into
    byte-range chunks at newline boundaries and proce
…[8578 chars omitted]…
_chunks = min(n_workers * 3, max_chunks)

    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
    n_chunks = len(offsets) - 1
    if n_chunks < 2:
        # e.g. a data section with no interior newlines (one giant line)
        return None

    # ------------------------------------------------------------------
    # Infer column names from the preamble + one data line (very fast).

low_memory=False: eligible False; result equals serial
500-column / 300-row file: serial fallback
large first row: estimated rows 32137; requested chunks 12; result equals serial

diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 97b77f2..c865e9f 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -183,6 +183,7 @@ _python_unsupported = {"low_memory", "float_precision"}
 # Minimum file size (bytes) to attempt parallel CSV reading.
 # Below this threshold the overhead of splitting and threading outweighs the benefit.
 _PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
+_PARALLEL_MIN_CHUNK_ROWS = 2_000
 _pyarrow_unsupported = {
     "skipfooter",
     "float_precision",

@@ -401,6 +402,8 @@ def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
       literal newline would be split mid-field.
     * ``parse_dates`` is unset - datetime format inference is per-chunk and may
       disagree with the serial whole-column inference.
+    * ``low_memory`` is not ``False`` - whole-file type inference can
…[3886 chars omitted]…
ath, data_start)
+    max_chunks = estimated_rows // _PARALLEL_MIN_CHUNK_ROWS
+    if max_chunks < 2:
+        return None
+    n_chunks = min(n_workers * 3, max_chunks)
+
+    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
     n_chunks = len(offsets) - 1
     if n_chunks < 2:
         # e.g. a data section with no interior newlines (one giant line)
 M pandas/io/parsers/readers.py

..............................................................................
============================= slowest 30 durations =============================
0.05s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_auto_parallel
0.05s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_parallel_vs_serial_large_file
0.02s setup    pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.02s call     pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.02s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_default_off_on_windows
0.02s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_default_thread_cap
0.01s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_ints_and_floats
0.01s call     pandas/tests/io
…[282 chars omitted]…
   pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_header_none

0.01s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_dict_dtype_duplicate_names_falls_back
0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_bom_bytes_mid_file_match_serial

(17 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 0.96s

/usr/local/bin/python: No module named ruff

diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 97b77f2..4cc6274 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -183,6 +183,8 @@ _python_unsupported = {"low_memory", "float_precision"}
 # Minimum file size (bytes) to attempt parallel CSV reading.
 # Below this threshold the overhead of splitting and threading outweighs the benefit.
 _PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
+# Minimum estimated number of data rows per parallel chunk.
+_PARALLEL_MIN_CHUNK_ROWS = 2_000
 _pyarrow_unsupported = {
     "skipfooter",

     "float_precision",
@@ -401,6 +403,8 @@ def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
       literal newline would be split mid-field.
     * ``parse_dates`` is unset - datetime format inference is per-chunk and may
       disagree with the serial whole-column inference.
+    * ``
…[4014 chars omitted]…
CHUNK_ROWS
+    if max_chunks < 2:
+        return None
+    n_chunks = min(n_workers * 3, max_chunks)
+
+    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
     n_chunks = len(offsets) - 1
     if n_chunks < 2:
         # e.g. a data section with no interior newlines (one giant line)
ERROR: file or directory not found: pandas/tests/io/parser/test_readers.py


no tests ran in 0.08s

pandas/tests/io/parser/__init__.py
pandas/tests/io/parser/conftest.py
pandas/tests/io/parser/test_c_parser_only.py
pandas/tests/io/parser/test_comment.py
pandas/tests/io/parser/test_compression.py
pandas/tests/io/parser/test_concatenate_chunks.py
pandas/tests/io/parser/test_converters.py
pandas/tests/io/parser/test_dialect.py
pandas/tests/io/parser/test_encoding.py
pandas/tests/io/parser/test_header.py
pandas/tests/io/parser/test_index_col.py
pandas/tests/io/parser/test_mangle_dupes.py

pandas/tests/io/parser/test_multi_thread.py
pandas/tests/io/parser/test_na_values.py
pandas/tests/io/parser/test_network.py
pandas/tests/io/parser/test_parallel_read.py
pandas/tests/io/parser/test_parse_dates.py
pandas/tests/io/parser/test_python_parser_only.py
pandas/tests/io/parser/test_quoting.py
pandas/tests/io/parser/test_read_fwf.py
pandas/tests/io/parser/test_skiprows.py
pandas/tests/io/parser/te
…[1277 chars omitted]…
andas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_dtype_specified
0.01s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_dict_dtype_duplicate_names_falls_back
0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_bom_bytes_mid_file_match_serial

(17 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 1.03s

M pandas/io/parsers/readers.py
helper default low_memory: True

M pandas/io/parsers/readers.py

......ss.........sss......ss...s...s......ss...s...s...s...s......ss...s...s...s.............xxsssss.........sss...s...ss..s...s......ss.........sss...s...s...s...s...s...s...s.........sss............ssss......ss...s...s.........sss...s......ss...s.........sss...s...s......ss...s...s...............sssss...s...s...s......ss...s...s...s...s...s...s....s...s...s...s......ss...sE
==================================== ERRORS ====================================
______________________ ERROR at setup of test_url[c_high] ______________________
file /repo/pandas/tests/io/parser/common/test_file_buffer_url.py, line 39
  @pytest.mark.network
  @pytest.mark.single_cpu
  def test_url(all_parsers, csv_dir_path, httpserver):
E       fixture 'httpserver' not found
>       available fixtures: add_doctest_imports, all_arithmetic_functions, all_arithmetic_operators, all_binary_operators, all_boolean_redu
…[3529 chars omitted]…
t_common_basic.py::test_read_csv_local[c_low]

(25 durations < 0.005s hidden.  Use -vv to show these durations.)
=========================== short test summary info ============================
ERROR pandas/tests/io/parser/common/test_file_buffer_url.py::test_url[c_high]
!!!!!!!!!!!!!!!!!!!!!!!!!! stopping after 1 failures !!!!!!!!!!!!!!!!!!!!!!!!!!!
280 passed, 95 skipped, 2 xfailed, 1 error in 4.05s

if not fd.readline():
                break
        return fd.tell()


def _estimate_csv_rows(filepath: str, data_start: int) -> int:
    """Estimate the number of data rows by sampling lines across the file."""
    file_size = os.path.getsize(filepath)
    data_size = file_size - data_start
    if data_size <= 0:
        return 0

    # Sample at evenly spaced byte positions, skip the partial line at each
    # position, and measure the following complete line.  Sampling multiple
    # positions (and not relying on the first data row) avoids letting an
    # unusually long first row dominate the estimate.
    sample_count = 10
    line_lengths = []
    with open(filepath, "rb") as fd:
        first_line_length = len(fd.readline())
        last_sample_end = data_start
        for i in range(1, sample_count + 1):
            target = data_start + data_size * i // (sample_co
…[1009 chars omitted]…
ian_line_length)


def _find_chunk_byte_offsets(
    filepath: str,
    n_chunks: int,
    data_start: int,
) -> list[int]:
    """
    Compute byte offsets that partition the data portion of *filepath* into
    *n_chunks* approximately equal pieces aligned to newline boundaries.

    Returns a list of ``n + 1`` offsets where the byte range
    ``[offsets[i], offsets[i+1])`` defines chunk *i*.
    """

low_memory=False: eligible False; result equals serial
500-column / 300-row file: serial fallback
large first row: estimated rows 28291; requested chunks 12; result equals serial
..............................................................................
============================= slowest 30 durations =============================
0.06s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_parallel_vs_serial_large_file
0.03s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_auto_parallel
0.02s call     pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.02s setup    pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.02s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_default_off_on_windows
0.01s call     pandas/tests/io/parser/test_paral
…[351 chars omitted]…

 call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_header_none
0.01s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_dtype_specified
0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_ragged_line_at_chunk_start_skip_matches_serial

(18 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 0.95s

......ss.........sss......ss...s...s......ss...s...s...s...s......ss...s...s...s.............xxsssss.........sss...s...ss..s...s......ss.........sss...s...s...s...s...s...s...s.........sss............ssss......ss...s...s.........sss...s......ss...s.........sss...s...s......ss...s...s...............sssss...s...s...s......ss...s...s...s...s...s...s....s...s...s...s......ss...s...s...s...sssss.................................sssssssssss...s...s...s...s...s...s...s...s............ssss......s......s......ssssssss...s...s...s...s...s.......s.......................................................................................................................sssssssssssssssss........................................................ssssssss.............................sss......ss......ss...s............ssss...s...s...s...s...s...s...s......ss......ss...s............ssss...s......ss...s........
…[6900 chars omitted]…
ary_mode_file_buffers[python-file_path1-latin-1]
0.01s call     pandas/tests/io/parser/dtypes/test_categorical.py::test_categorical_dtype_utf16[python]
0.01s call     pandas/tests/io/parser/test_encoding.py::test_binary_mode_file_buffers[c_high-file_path0-utf-8]
0.01s setup    pandas/tests/io/parser/test_compression.py::test_zip[python-zip]
3977 passed, 925 skipped, 60 deselected, 52 xfailed in 24.08s

pandas/io/parsers/readers.py | 72 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 68 insertions(+), 4 deletions(-)
 M pandas/io/parsers/readers.py
diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 97b77f2..56e84b5 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -183,6 +183,8 @@ _python_unsupported = {"low_memory", "float_precision"}
 # Minimum file size (bytes) to attempt parallel CSV reading.
 # Below this threshold the overhead of splitting and threading outweighs the benefit.
 _PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
+# Minimum estimated number of data rows per parallel chunk.

+_PARALLEL_MIN_CHUNK_ROWS = 2_000
 _pyarrow_unsupported = {
     "skipfooter",
     "float_precision",
@@ -401,6 +403,8 @@ def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
       literal newline would be s
…[4479 chars omitted]…
_rows = _estimate_csv_rows(filepath, data_start)
+    max_chunks = estimated_rows // _PARALLEL_MIN_CHUNK_ROWS
+    if max_chunks < 2:
+        return None
+    n_chunks = min(n_workers * 3, max_chunks)
+
+    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
     n_chunks = len(offsets) - 1
     if n_chunks < 2:
         # e.g. a data section with no interior newlines (one giant line)

M pandas/io/parsers/readers.py
 pandas/io/parsers/readers.py | 72 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 68 insertions(+), 4 deletions(-)

…[14 chars omitted]…
nput fell back to serial. test_parallel_read.py passed (78 tests); parser suite excluding network tests passed (3977 passed, 925 skipped, 60 deselected, 52 xfailed)."}
submitted.

da9e5ee base
185:_PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
347:        if _n_workers > 1 and _can_parallelize_csv(filepath_or_buffer, kwds):
349:            assert isinstance(_filepath, str)  # guaranteed by _can_parallelize_csv
351:                result = _read_csv_parallel(_filepath, kwds, _n_workers)
371:def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
408:    * The file is at least ``_PARALLEL_READ_MIN_BYTES`` bytes large.
534:        if file_size < _PARALLEL_READ_MIN_BYTES:
540:    # enough to split into at least 2 chunks.  Without this, _read_csv_parallel
547:    if file_size - data_start < _PARALLEL_READ_MIN_BYTES:
586:def _find_chunk_byte_offsets(
623:def _read_csv_parallel(

639:    The caller must ensure the file is eligible via :func:`_can_parallelize_csv`
661:    offsets = _find_chunk_byte_offsets(filepath, n_workers, data_start)

low_memory: Literal[True]
    memory_map: Literal[False]
    float_precision: None


_c_parser_defaults: _C_Parser_Defaults = {
    "na_filter": True,
    "low_memory": True,
    "memory_map": False,
    "float_precision": None,
}

class _Fwf_Defaults(TypedDict):
    colspecs: Literal["infer"]
    infer_nrows: Literal[100]
    widths: None


_fwf_defaults: _Fwf_Defaults = {"colspecs": "infer", "infer_nrows": 100, "widths": None}
_c_unsupported = {"skipfooter"}
_python_unsupported = {"low_memory", "float_precision"}

# Minimum file size (bytes) to attempt parallel CSV reading.
# Below this threshold the overhead of splitting and threading outweighs the benefit.
_PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
_pyarrow_unsupported = {
    "skipfooter",
    "float_precision",
    "chunksize",
    "comment",
    "nrows",
    "thousands",
    "memory_map",
    "dialect",
    "quot
…[8629 chars omitted]…
ol_names: list = list(name_reader._engine.orig_names)
    # name_buf holds the preamble plus exactly one data line.  Parsing it must
    # yield exactly one row; anything else means the preamble's physical line
    # count disagrees with its logical row count (e.g. a quoted embedded
    # newline in the header), so data_start is wrong.
    n_first_rows = len(name_reader.read())
    name_reader.close()

def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
    """
    Return True when a ``read_csv`` call is eligible for parallel execution.

    Parallel reading works by splitting the file's data section into
    byte-range chunks at newline boundaries and processing them concurrently
    with threads.  The following conditions must all hold:

    * *filepath_or_buffer* is a local file path (not a URL or file object).
    * The file is not compressed.
    * The C engine is selected (``engine='c'``, the default).
    * The call is not in iterator / chunked mode.

    * The entire file is being read (no ``nrows`` limit).
    * ``skiprows`` is a plain integer (or absent) - list/callable forms require
      tracking absolute line numbers across chunks.
    * ``header`` is a single integer or ``None`` - multi-level headers complicate
      the preamble boundary.
    * ``index_co
…[7916 chars omitted]…
 is the byte from which each non-first chunk
    must begin.
    """
    if header is None:
        header_lines = 0
    else:
        header_lines = int(header) + 1  # header=0 → 1 header line

    preamble_lines = int(skiprows) + header_lines
    with open(filepath, "rb") as fd:
        for _ in range(preamble_lines):
            if not fd.readline():
                break
        return fd.tell()

name_reader.close()
    if n_first_rows != 1:
        return None

    # A dict ``dtype`` is applied per raw header name: when a name is repeated,
    # the serial path assigns that dtype to every de-duplicated column (``a``
    # and ``a.1``), but the workers receive the already-de-duplicated
    # ``col_names`` and would match only ``a``.  ``col_names`` no longer shows
    # the duplication, so recover the raw header names (parse the header line
    # as data) and fall back to serial when they collide.  GH#64347
    if header is not None and isinstance(kwds.get("dtype"), dict):
        header_line = preamble.splitlines(keepends=True)[-1]

        raw_reader = TextFileReader(
            io.BytesIO(header_line),
            **{
                **base_kwds,
                "header": None,
                "names": None,
                "usecols": None,
                "index_col": F
…[6050 chars omitted]…
["low_memory"],
    memory_map: bool = False,
    float_precision: Literal["high", "legacy", "round_trip"] | None = None,
    storage_options: StorageOptions | None = None,
    dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
) -> DataFrame | TextFileReader:
    """
    Read a comma-separated values (csv) file into DataFrame.

    Also supports optionally iterating or breaking of the file

pandas/tests/io/parser/test_parallel_read.py:2:Tests for the parallel read_csv implementation (C engine, large local files).
pandas/tests/io/parser/test_parallel_read.py:4:The parallel path is enabled automatically when:
pandas/tests/io/parser/test_parallel_read.py:10:We test correctness (parallel == serial) rather than performance.
pandas/tests/io/parser/test_parallel_read.py:40:    _can_parallelize_csv,
pandas/tests/io/parser/test_parallel_read.py:41:    _find_chunk_byte_offsets,
pandas/tests/io/parser/test_parallel_read.py:43:    _read_csv_parallel,
pandas/tests/io/parser/test_parallel_read.py:52:    """Write a CSV with enough rows to split into multiple parallel chunks."""
pandas/tests/io/parser/test_parallel_read.py:65:# _can_parallelize_csv
pandas/tests/io/parser/test_parallel_read.py:96:        assert _can_parallelize_csv(path, self._kwds())
pandas/tests/io/parser/test_parallel
…[7876 chars omitted]…
05:        assert _read_csv_parallel(str(path), kwds, 4) is not None

pandas/tests/io/parser/test_parallel_read.py:642:    as a direct serial read.  The parallel path either parses correctly (when
pandas/tests/io/parser/test_parallel_read.py:651:    parallel = read_csv(path)  # may take parallel path or fall back
pandas/tests/io/parser/test_parallel_read.py:653:    assert len(parallel) == expected_rows

"""
Tests for the parallel read_csv implementation (C engine, large local files).

The parallel path is enabled automatically when:
  - reading from a local uncompressed file path
  - using the C engine (the default)
  - not in iterator / chunked mode
  - file size >= _PARALLEL_READ_MIN_BYTES

We test correctness (parallel == serial) rather than performance.
"""

from __future__ import annotations

import csv
import io
import os
from typing import TYPE_CHECKING

import numpy as np
import pytest

from pandas.errors import (
    ParserError,
    ParserWarning,
)

if TYPE_CHECKING:
    from pathlib import Path

from pandas import (
    DataFrame,
    option_context,
    read_csv,
)
import pandas._testing as tm

from pandas.io.parsers.base_parser import ParserBase
from pandas.io.parsers.readers import (
    _can_parallelize_csv,
    _find_chunk_byte_offsets,
    _find_data_start_offset,
 
…[8629 chars omitted]…
l path.
    monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)

    # Force the parallel path regardless of the platform default (it is off by
    # default on Windows) so this exercises parallel == serial on every platform.
    with option_context("mode.max_threads", 4):
        result = read_csv(path)  # auto-selects parallel path for C engine
    expected = read_csv(path, engine="python")

# _read_csv_parallel  (correctness: parallel == serial)
# ---------------------------------------------------------------------------


@pytest.mark.slow
class TestReadCsvParallel:
    """End-to-end correctness: parallel result must match serial result."""

    def _serial_read(self, path, **kwargs):
        return read_csv(path, engine="c", **kwargs)

    def _parallel_read(self, path, kwds, n_workers=4):

        """Call the internal helper directly so file-size guards don't apply."""
        result = _read_csv_parallel(str(path), kwds, n_workers)
        if result is None:
            pytest.skip("parallel read not applicable to this file")
        return result

    def _base_kwds(self, path, **overrides):
        """Return kwds as _read() would see them for a default read_csv call."""
        from pandas._libs import lib

        from pandas.io.parsers.readers import _refine_defau
…[6884 chars omitted]…
-------------------------------------------------------
# Embedded newlines in quoted fields
# ---------------------------------------------------------------------------


@pytest.mark.parametrize(
    "csv_bytes,expected_rows",
    [
        # Simple embedded newline
        pytest.param(
            b'a,b,c\n1,"hello\nworld",3\n4,normal,6\n',
            2,
            id="simple_embedded_newline",

def _validate_names(names: Sequence[Hashable] | None) -> None:
    """
    Raise ValueError if the `names` parameter contains duplicates or has an
    invalid data type.

    Parameters
    ----------
    names : array-like or None
        An array containing a list of the names used for the output DataFrame.

    Raises
    ------

    ValueError
        If names are not unique or are not ordered (e.g. set).
    """
    if names is not None:
        if len(names) != len(set(names)):
            raise ValueError("Duplicate names are not allowed.")
        if not (
            is_list_like(names, allow_sets=False) or isinstance(names, abc.KeysView)
        ):
            raise ValueError("Names should be an ordered collection.")


def _read(
    filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], kwds
) -> DataFrame | TextFileReader:
    """Generic reader of line f
…[3885 chars omitted]…
       low_memory: bool
160:    low_memory: Literal[True]
167:    "low_memory": True,
181:_python_unsupported = {"low_memory", "float_precision"}
201:    "low_memory",
891:    low_memory: bool = _c_parser_defaults["low_memory"],
1225:    low_memory : bool, default True
1485:    low_memory: bool = _c_parser_defaults["low_memory"],
1815:    low_memory : bool, default True
2806:def _refine_defaults_read(

pandas/io/parsers/readers.py | 50 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 48 insertions(+), 2 deletions(-)
diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 97b77f2..91d98d6 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -183,6 +183,9 @@ _python_unsupported = {"low_memory", "float_precision"}
 # Minimum file size (bytes) to attempt parallel CSV reading.
 # Below this threshold the overhead of splitting and threading outweighs the benefit.
 _PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
+# Minimum number of data rows per parallel chunk.  Splitting a file into chunks
+# smaller than this costs more in per-chunk overhead than it saves.

+_PARALLEL_MIN_CHUNK_ROWS = 2000
 _pyarrow_unsupported = {
     "skipfooter",
     "float_precision",
@@ -405,6 +408,9 @@ def _can_parallelize_csv(filepath_or_buffer
…[2820 chars omitted]…
# single slow chunk cannot dominate the wall-clock time, but never create
+    # chunks below the row floor.
+    n_chunks = min(
+        n_workers * 3, estimated_rows // _PARALLEL_MIN_CHUNK_ROWS
+    )
+    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
     n_chunks = len(offsets) - 1
     if n_chunks < 2:
         # e.g. a data section with no interior newlines (one giant line)

..............................................................................
============================= slowest 30 durations =============================
0.07s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_auto_parallel
0.05s call     pandas/tests/io/parser/test_parallel_read.py::test_read_csv_parallel_vs_serial_large_file
0.04s call     pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.03s setup    pandas/tests/io/parser/test_parallel_read.py::TestCanParallelizeCsv::test_eligible_large_file
0.02s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_ints_and_floats
0.02s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_default_off_on_windows
0.02s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_default_thread_cap
0.01s call     pandas/tests/io
…[618 chars omitted]…
ndas/tests/io/parser/test_parallel_read.py::test_parallel_bom_at_file_start_header_none

0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_ragged_line_at_chunk_start_skip_matches_serial
0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_ragged_line_at_chunk_start_raises

(14 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 1.40s

can parallel low_memory=False: False
Traceback (most recent call last):
  File "<stdin>", line 23, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 1436, in read_csv
    return _read(filepath_or_buffer, kwds)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 365, in _read
    parser = TextFileReader(filepath_or_buffer, **kwds)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2261, in __init__
    options = self._get_options_with_defaults(engine)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

  File "/repo/pandas/io/parsers/readers.py", line 2329, in _get_options_with_defaults
    raise ValueError(
ValueError: The 'low_memory' option is not supported with the 'python' engine

can parallel low_memory=False: False
low_memory=False matches serial
wide 300 rows parallel: None
huge first requested chunks: [12] result rows: None
Traceback (most recent call last):
  File "<stdin>", line 47, in <module>
AttributeError: 'NoneType' object has no attribute 'reset_index'

calls [12] res None
Traceback (most recent call last):
  File "<stdin>", line 22, in <module>
AttributeError: 'NoneType' object has no attribute 'reset_index'

data_start 4 size 447787
offsets [4, 50007, 74643, 111951, 149271, 186589, 223899, 261223, 298533, 335843, 373167, 410477, 447787]
chunk sizes [50003, 24636, 37308, 37320, 37318, 37310, 37324, 37310, 37310, 37324, 37310, 37310]
res None

Traceback (most recent call last):
  File "<stdin>", line 17, in <module>
AttributeError: 'PythonParser' object has no attribute '_reader'. Did you mean: 'header'?

Traceback (most recent call last):
  File "<stdin>", line 17, in <module>
  File "/repo/pandas/io/parsers/readers.py", line 728, in _read_csv_parallel
    name_reader = TextFileReader(name_buf, **base_kwds)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2274, in __init__
    self._engine = self._make_engine(f, self.engine)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/readers.py", line 2550, in _make_engine
    return mapping[engine](f, **self.options)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/repo/pandas/io/parsers/c_parser_wrapper.py", line 79, in __init__

    kwds["on_bad_lines"] = self.on_bad_lines.value
                           ^^^^^^^^^^^^^^^^^^^^^^^
AttributeError: 'str' object has no attribute 'value'

calls [12] res None
Traceback (most recent call last):
  File "<stdin>", line 24, in <module>
AttributeError: 'NoneType' object has no attribute 'reset_index'

engine <class 'pandas.io.parsers.c_parser_wrapper.CParserWrapper'> n_first_rows 1 leading 0
orig_names ['a', 'b']
0 (1, 2) [<StringDtype(storage='python', na_value=nan)>, dtype('int64')] ['xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
…[8630 chars omitted]…
] ['v22004', 'w22004']
10 (2665, 2) [<StringDtype(storage='python', na_value=nan)>, <StringDtype(storage='python', na_value=nan)>] ['v24670', 'w24670']
11 (2665, 2) [<StringDtype(storage='python', na_value=nan)>, <StringDtype(storage='python', na_value=nan)>] ['v27335', 'w27335']
Traceback (most recent call last):
  File "<stdin>", line 32, in <module>
BufferError: cannot close exported pointers exist

Refused: `find /` walks the whole container FS. Scope under /repo.

* master
/repo/pandas/tests/io/parser/test_parallel_read.py

calls [12] res 30001
ok

/usr/local/bin/python: No module named ruff
/usr/local/bin/python: No module named ruff

0.01s call     pandas/tests/io/parser/test_parallel_read.py::TestReadCsvParallel::test_skiprows_int
0.01s call     pandas/tests/io/parser/test_parallel_read.py::test_parallel_ragged_line_at_chunk_start_raises

(13 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 1.49s

diff --git a/pandas/io/parsers/readers.py b/pandas/io/parsers/readers.py
index 97b77f2..69f3a5e 100644
--- a/pandas/io/parsers/readers.py
+++ b/pandas/io/parsers/readers.py
@@ -183,6 +183,9 @@ _python_unsupported = {"low_memory", "float_precision"}
 # Minimum file size (bytes) to attempt parallel CSV reading.
 # Below this threshold the overhead of splitting and threading outweighs the benefit.
 _PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
+# Minimum number of data rows per parallel chunk.  Splitting a file into chunks
+# smaller than this costs more in per-chunk overhead than it saves.
+_PARALLEL_MIN_CHUNK_ROWS = 2000
 _pyarrow_unsupported = {

     "skipfooter",
     "float_precision",
@@ -405,6 +408,9 @@ def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
       path, and that error must not be masked.
     * ``on_bad_lines`` is not ``"warn"`` - chunk workers w
…[2674 chars omitted]…
 so that a
+    # single slow chunk cannot dominate the wall-clock time, but never create
+    # chunks below the row floor.
+    n_chunks = min(n_workers * 3, estimated_rows // _PARALLEL_MIN_CHUNK_ROWS)
+    offsets = _find_chunk_byte_offsets(filepath, n_chunks, data_start)
     n_chunks = len(offsets) - 1
     if n_chunks < 2:
         # e.g. a data section with no interior newlines (one giant line)

ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[gzip-c-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[bz2-python-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[bz2-python-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[bz2-c-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[bz2-c-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[zip-python-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[zip-python-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[zip-c-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[zip-c-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[xz-python-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compre
…[309 chars omitted]…

_network.py::test_compressed_urls[tar-python-infer]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-c-explicit]
ERROR pandas/tests/io/parser/test_network.py::test_compressed_urls[tar-c-infer]
ERROR pandas/tests/io/parser/test_network.py::test_url_encoding_csv
ERROR pandas/tests/io/parser/test_read_fwf.py::test_url_urlopen
3977 passed, 960 skipped, 52 xfailed, 25 errors in 36.61s

./pandas/io/parsers/readers.py:185:_PARALLEL_READ_MIN_BYTES = 50 * 1024 * 1024  # 50 MB
./pandas/io/parsers/readers.py:188:_PARALLEL_MIN_CHUNK_ROWS = 2000
./pandas/io/parsers/readers.py:350:        if _n_workers > 1 and _can_parallelize_csv(filepath_or_buffer, kwds):
./pandas/io/parsers/readers.py:352:            assert isinstance(_filepath, str)  # guaranteed by _can_parallelize_csv
./pandas/io/parsers/readers.py:354:                result = _read_csv_parallel(_filepath, kwds, _n_workers)
./pandas/io/parsers/readers.py:374:def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
./pandas/io/parsers/readers.py:414:    * The file is at least ``_PARALLEL_READ_MIN_BYTES`` bytes large.
./pandas/io/parsers/readers.py:546:        if file_size < _PARALLEL_READ_MIN_BYTES:
./pandas/io/parsers/readers.py:552:    # enough to split into at least 2 chunks.  Without this, _read_csv_paralle
…[181 chars omitted]…
.py:651:    The caller must ensure the file is eligible via :func:`_can_parallelize_csv`
./pandas/io/parsers/readers.py:696:    # A 2-way split must still leave at least _PARALLEL_MIN_CHUNK_ROWS rows in

./pandas/io/parsers/readers.py:698:    if estimated_rows < 2 * _PARALLEL_MIN_CHUNK_ROWS:
./pandas/io/parsers/readers.py:704:    n_chunks = min(n_workers * 3, estimated_rows // _PARALLEL_MIN_CHUNK_ROWS)

serial-fallback signal too.
    """
    # Resolve the effective header value (mirrors TextFileReader.__init__).
    header = kwds.get("header", "infer")
    if header == "infer":
        header = 0 if kwds.get("names") is None else None

    skiprows = int(kwds.get("skiprows") or 0)

    # Byte offset at which real data rows begin.
    data_start = _find_data_start_offset(filepath, header, skiprows)

    # Estimate the number of data rows by sampling lines spread across the
    # data section.  Using only the first data line is unreliable: a single
    # unusually large row would make the file look tiny and collapse the
    # split.  Sampling several lines gives a representative average row size.
    file_size = os.path.getsize(filepath)
    data_size = file_size - data_start
    n_samples = 5
    sampled_bytes = 0
    sampled_rows = 0
    with open(filepath, "rb") as fd:
        
…[1116 chars omitted]…
    n_chunks = len(offsets) - 1
    if n_chunks < 2:
        # e.g. a data section with no interior newlines (one giant line)
        return None

    # ------------------------------------------------------------------
    # Infer column names from the preamble + one data line (very fast).
    # ------------------------------------------------------------------
    base_kwds: dict = {
        **kwds,

False False
True True
None True

(13 durations < 0.005s hidden.  Use -vv to show these durations.)
78 passed in 1.53s

…[144 chars omitted]…
unks bounded by the estimated row count.\n- Verified existing parallel-read tests (78 passed) and reproduced the low_memory, wide-file, and huge-first-row scenarios."}
submitted.