Инструменты ввода-вывода (текст, CSV, HDF5 и т. д.)

In [13]: import numpy as np

In [14]: data = "a,b,c,d\n1,2,3,4\n5,6,7,8\n9,10,11"

In [15]: print(data)
a,b,c,d
1,2,3,4
5,6,7,8
9,10,11

In [16]: df = pd.read_csv(StringIO(data), dtype=object)

In [17]: df
Out[17]: 
   a   b   c    d
0  1   2   3    4
1  5   6   7    8
2  9  10  11  NaN

In [18]: df["a"][0]
Out[18]: '1'

In [19]: df = pd.read_csv(StringIO(data), dtype={"b": object, "c": np.float64, "d": "Int64"})

In [20]: df.dtypes
Out[20]: 
a      int64
b     object
c    float64
d      Int64
dtype: object

In [21]: data = "col_1\n1\n2\n'A'\n4.22"

In [22]: df = pd.read_csv(StringIO(data), converters={"col_1": str})

In [23]: df
Out[23]: 
  col_1
0     1
1     2
2   'A'
3  4.22

In [24]: df["col_1"].apply(type).value_counts()
Out[24]: 
<class 'str'>    4
Name: col_1, dtype: int64

In [25]: df2 = pd.read_csv(StringIO(data))

In [26]: df2["col_1"] = pd.to_numeric(df2["col_1"], errors="coerce")

In [27]: df2
Out[27]: 
   col_1
0   1.00
1   2.00
2    NaN
3   4.22

In [28]: df2["col_1"].apply(type).value_counts()
Out[28]: 
<class 'float'>    4
Name: col_1, dtype: int64

In [35]: data = "col1,col2,col3\na,b,1\na,b,2\nc,d,3"

In [36]: pd.read_csv(StringIO(data))
Out[36]: 
  col1 col2  col3
0    a    b     1
1    a    b     2
2    c    d     3

In [37]: pd.read_csv(StringIO(data)).dtypes
Out[37]: 
col1    object
col2    object
col3     int64
dtype: object

In [38]: pd.read_csv(StringIO(data), dtype="category").dtypes
Out[38]: 
col1    category
col2    category
col3    category
dtype: object

In [39]: pd.read_csv(StringIO(data), dtype={"col1": "category"}).dtypes
Out[39]: 
col1    category
col2      object
col3       int64
dtype: object

In [40]: from pandas.api.types import CategoricalDtype

In [41]: dtype = CategoricalDtype(["d", "c", "b", "a"], ordered=True)

In [42]: pd.read_csv(StringIO(data), dtype={"col1": dtype}).dtypes
Out[42]: 
col1    category
col2      object
col3       int64
dtype: object

In [43]: dtype = CategoricalDtype(["a", "b", "d"])  # No 'c'

In [44]: pd.read_csv(StringIO(data), dtype={"col1": dtype}).col1
Out[44]: 
0      a
1      a
2    NaN
Name: col1, dtype: category
Categories (3, object): ['a', 'b', 'd']

In [59]: data = "a,b,a\n0,1,2\n3,4,5"

In [60]: pd.read_csv(StringIO(data))
Out[60]: 
   a  b  a.1
0  0  1    2
1  3  4    5

In [84]: from io import BytesIO

In [85]: data = b"word,length\n" b"Tr\xc3\xa4umen,7\n" b"Gr\xc3\xbc\xc3\x9fe,5"

In [86]: data = data.decode("utf8").encode("latin-1")

In [87]: df = pd.read_csv(BytesIO(data), encoding="latin-1")

In [88]: df
Out[88]: 
      word  length
0  Träumen       7
1    Grüße       5

In [89]: df["word"][1]
Out[89]: 'Grüße'

In [90]: data = "a,b,c\n4,apple,bat,5.7\n8,orange,cow,10"

In [91]: pd.read_csv(StringIO(data))
Out[91]: 
        a    b     c
4   apple  bat   5.7
8  orange  cow  10.0

In [92]: data = "index,a,b,c\n4,apple,bat,5.7\n8,orange,cow,10"

In [93]: pd.read_csv(StringIO(data), index_col=0)
Out[93]: 
            a    b     c
index                   
4       apple  bat   5.7
8      orange  cow  10.0

In [94]: data = "a,b,c\n4,apple,bat,\n8,orange,cow,"

In [95]: print(data)
a,b,c
4,apple,bat,
8,orange,cow,

In [96]: pd.read_csv(StringIO(data))
Out[96]: 
        a    b   c
4   apple  bat NaN
8  orange  cow NaN

In [97]: pd.read_csv(StringIO(data), index_col=False)
Out[97]: 
   a       b    c
0  4   apple  bat
1  8  orange  cow

In [98]: data = "a,b,c\n4,apple,bat,\n8,orange,cow,"

In [99]: print(data)
a,b,c
4,apple,bat,
8,orange,cow,

In [100]: pd.read_csv(StringIO(data), usecols=["b", "c"])
Out[100]: 
     b   c
4  bat NaN
8  cow NaN

In [101]: pd.read_csv(StringIO(data), usecols=["b", "c"], index_col=0)
Out[101]: 
     b   c
4  bat NaN
8  cow NaN

In [136]: val = "0.3066101993807095471566981359501369297504425048828125"

In [137]: data = "a,b,c\n1,2,{0}".format(val)

In [138]: abs(
   .....:     pd.read_csv(
   .....:         StringIO(data),
   .....:         engine="c",
   .....:         float_precision=None,
   .....:     )["c"][0] - float(val)
   .....: )
   .....: 
Out[138]: 5.551115123125783e-17

In [139]: abs(
   .....:     pd.read_csv(
   .....:         StringIO(data),
   .....:         engine="c",
   .....:         float_precision="high",
   .....:     )["c"][0] - float(val)
   .....: )
   .....: 
Out[139]: 5.551115123125783e-17

In [140]: abs(
   .....:     pd.read_csv(StringIO(data), engine="c", float_precision="round_trip")["c"][0]
   .....:     - float(val)
   .....: )
   .....: 
Out[140]: 0.0

In [141]: data = (
   .....:     "ID|level|category\n"
   .....:     "Patient1|123,000|x\n"
   .....:     "Patient2|23,000|y\n"
   .....:     "Patient3|1,234,018|z"
   .....: )
   .....: 

In [142]: with open("tmp.csv", "w") as fh:
   .....:     fh.write(data)
   .....: 

In [143]: df = pd.read_csv("tmp.csv", sep="|")

In [144]: df
Out[144]: 
         ID      level category
0  Patient1    123,000        x
1  Patient2     23,000        y
2  Patient3  1,234,018        z

In [145]: df.level.dtype
Out[145]: dtype('O')

In [146]: df = pd.read_csv("tmp.csv", sep="|", thousands=",")

In [147]: df
Out[147]: 
         ID    level category
0  Patient1   123000        x
1  Patient2    23000        y
2  Patient3  1234018        z

In [148]: df.level.dtype
Out[148]: dtype('int64')

In [149]: data = "level\nPatient1,123000\nPatient2,23000\nPatient3,1234018"

In [150]: with open("tmp.csv", "w") as fh:
   .....:     fh.write(data)
   .....: 

In [151]: print(open("tmp.csv").read())
level
Patient1,123000
Patient2,23000
Patient3,1234018

In [152]: output = pd.read_csv("tmp.csv", squeeze=True)

In [153]: output
Out[153]: 
Patient1     123000
Patient2      23000
Patient3    1234018
Name: level, dtype: int64

In [154]: type(output)
Out[154]: pandas.core.series.Series

In [155]: data = "a,b,c\n1,Yes,2\n3,No,4"

In [156]: print(data)
a,b,c
1,Yes,2
3,No,4

In [157]: pd.read_csv(StringIO(data))
Out[157]: 
   a    b  c
0  1  Yes  2
1  3   No  4

In [158]: pd.read_csv(StringIO(data), true_values=["Yes"], false_values=["No"])
Out[158]: 
   a      b  c
0  1   True  2
1  3  False  4

In [159]: data = "a,b,c\n1,2,3\n4,5,6,7\n8,9,10"

In [160]: pd.read_csv(StringIO(data))
---------------------------------------------------------------------------
ParserError                               Traceback (most recent call last)
Cell In [160], line 1
----> 1 pd.read_csv(StringIO(data))

File ~/work/pandas/pandas/pandas/util/_decorators.py:211, in deprecate_kwarg.<locals>._deprecate_kwarg.<locals>.wrapper(*args, **kwargs)
    209     else:
    210         kwargs[new_arg_name] = new_arg_value
--> 211 return func(*args, **kwargs)

File ~/work/pandas/pandas/pandas/util/_decorators.py:317, in deprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper(*args, **kwargs)
    311 if len(args) > num_allow_args:
    312     warnings.warn(
    313         msg.format(arguments=arguments),
    314         FutureWarning,
    315         stacklevel=find_stack_level(inspect.currentframe()),
    316     )
--> 317 return func(*args, **kwargs)

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:950, in read_csv(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)
    935 kwds_defaults = _refine_defaults_read(
    936     dialect,
    937     delimiter,
   (...)
    946     defaults={"delimiter": ","},
    947 )
    948 kwds.update(kwds_defaults)
--> 950 return _read(filepath_or_buffer, kwds)

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:611, in _read(filepath_or_buffer, kwds)
    608     return parser
    610 with parser:
--> 611     return parser.read(nrows)

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:1772, in TextFileReader.read(self, nrows)
   1765 nrows = validate_integer("nrows", nrows)
   1766 try:
   1767     # error: "ParserBase" has no attribute "read"
   1768     (
   1769         index,
   1770         columns,
   1771         col_dict,
-> 1772     ) = self._engine.read(  # type: ignore[attr-defined]
   1773         nrows
   1774     )
   1775 except Exception:
   1776     self.close()

File ~/work/pandas/pandas/pandas/io/parsers/c_parser_wrapper.py:243, in CParserWrapper.read(self, nrows)
    241 try:
    242     if self.low_memory:
--> 243         chunks = self._reader.read_low_memory(nrows)
    244         # destructive to chunks
    245         data = _concatenate_chunks(chunks)

File ~/work/pandas/pandas/pandas/_libs/parsers.pyx:808, in pandas._libs.parsers.TextReader.read_low_memory()

File ~/work/pandas/pandas/pandas/_libs/parsers.pyx:866, in pandas._libs.parsers.TextReader._read_rows()

File ~/work/pandas/pandas/pandas/_libs/parsers.pyx:852, in pandas._libs.parsers.TextReader._tokenize_rows()

File ~/work/pandas/pandas/pandas/_libs/parsers.pyx:1973, in pandas._libs.parsers.raise_parser_error()

ParserError: Error tokenizing data. C error: Expected 3 fields in line 3, saw 4

In [29]: pd.read_csv(StringIO(data), on_bad_lines="warn")
Skipping line 3: expected 3 fields, saw 4

Out[29]:
   a  b   c
0  1  2   3
1  8  9  10

In [29]: external_list = []

In [30]: def bad_lines_func(line):
    ...:     external_list.append(line)
    ...:     return line[-3:]

In [31]: pd.read_csv(StringIO(data), on_bad_lines=bad_lines_func, engine="python")
Out[31]:
   a  b   c
0  1  2   3
1  5  6   7
2  8  9  10

In [32]: external_list
Out[32]: [4, 5, 6, 7]

.. versionadded:: 1.4.0

In [33]: pd.read_csv(StringIO(data), usecols=[0, 1, 2])

 Out[33]:
    a  b   c
 0  1  2   3
 1  4  5   6
 2  8  9  10

In [34]: pd.read_csv(StringIO(data), names=['a', 'b', 'c', 'd'])

Out[34]:
    a  b   c  d
 0  1  2   3  NaN
 1  4  5   6  7
 2  8  9  10  NaN

In [161]: data = "label1,label2,label3\n" 'index1,"a,c,e\n' "index2,b,d,f"

In [162]: print(data)
label1,label2,label3
index1,"a,c,e
index2,b,d,f

In [163]: import csv

In [164]: dia = csv.excel()

In [165]: dia.quoting = csv.QUOTE_NONE

In [166]: pd.read_csv(StringIO(data), dialect=dia)
Out[166]: 
       label1 label2 label3
index1     "a      c      e
index2      b      d      f

In [167]: data = "a,b,c~1,2,3~4,5,6"

In [168]: pd.read_csv(StringIO(data), lineterminator="~")
Out[168]: 
   a  b  c
0  1  2  3
1  4  5  6

In [169]: data = "a, b, c\n1, 2, 3\n4, 5, 6"

In [170]: print(data)
a, b, c
1, 2, 3
4, 5, 6

In [171]: pd.read_csv(StringIO(data), skipinitialspace=True)
Out[171]: 
   a  b  c
0  1  2  3
1  4  5  6

In [172]: data = 'a,b\n"hello, \\"Bob\\", nice to see you",5'

In [173]: print(data)
a,b
"hello, \"Bob\", nice to see you",5

In [174]: pd.read_csv(StringIO(data), escapechar="\\")
Out[174]: 
                               a  b
0  hello, "Bob", nice to see you  5

In [175]: data1 = (
   .....:     "id8141    360.242940   149.910199   11950.7\n"
   .....:     "id1594    444.953632   166.985655   11788.4\n"
   .....:     "id1849    364.136849   183.628767   11806.2\n"
   .....:     "id1230    413.836124   184.375703   11916.8\n"
   .....:     "id1948    502.953953   173.237159   12468.3"
   .....: )
   .....: 

In [176]: with open("bar.csv", "w") as f:
   .....:     f.write(data1)
   .....: 

# Column specifications are a list of half-intervals
In [177]: colspecs = [(0, 6), (8, 20), (21, 33), (34, 43)]

In [178]: df = pd.read_fwf("bar.csv", colspecs=colspecs, header=None, index_col=0)

In [179]: df
Out[179]: 
                 1           2        3
0                                      
id8141  360.242940  149.910199  11950.7
id1594  444.953632  166.985655  11788.4
id1849  364.136849  183.628767  11806.2
id1230  413.836124  184.375703  11916.8
id1948  502.953953  173.237159  12468.3

# Widths are a list of integers
In [180]: widths = [6, 14, 13, 10]

In [181]: df = pd.read_fwf("bar.csv", widths=widths, header=None)

In [182]: df
Out[182]: 
        0           1           2        3
0  id8141  360.242940  149.910199  11950.7
1  id1594  444.953632  166.985655  11788.4
2  id1849  364.136849  183.628767  11806.2
3  id1230  413.836124  184.375703  11916.8
4  id1948  502.953953  173.237159  12468.3

In [183]: df = pd.read_fwf("bar.csv", header=None, index_col=0)

In [184]: df
Out[184]: 
                 1           2        3
0                                      
id8141  360.242940  149.910199  11950.7
id1594  444.953632  166.985655  11788.4
id1849  364.136849  183.628767  11806.2
id1230  413.836124  184.375703  11916.8
id1948  502.953953  173.237159  12468.3

In [185]: pd.read_fwf("bar.csv", header=None, index_col=0).dtypes
Out[185]: 
1    float64
2    float64
3    float64
dtype: object

In [186]: pd.read_fwf("bar.csv", header=None, dtype={2: "object"}).dtypes
Out[186]: 
0     object
1    float64
2     object
3    float64
dtype: object

In [208]: df = pd.DataFrame(np.random.randn(10, 4))

In [209]: df.to_csv("tmp.csv", sep="|")

In [210]: df.to_csv("tmp2.csv", sep=":")

In [211]: pd.read_csv("tmp2.csv", sep=None, engine="python")
Out[211]: 
   Unnamed: 0         0         1         2         3
0           0  0.469112 -0.282863 -1.509059 -1.135632
1           1  1.212112 -0.173215  0.119209 -1.044236
2           2 -0.861849 -2.104569 -0.494929  1.071804
3           3  0.721555 -0.706771 -1.039575  0.271860
4           4 -0.424972  0.567020  0.276232 -1.087401
5           5 -0.673690  0.113648 -1.478427  0.524988
6           6  0.404705  0.577046 -1.715002 -1.039268
7           7 -0.370647 -1.157892 -1.344312  0.844885
8           8  1.075770 -0.109050  1.643563 -1.469388
9           9  0.357021 -0.674600 -1.776904 -0.968914

In [212]: df = pd.DataFrame(np.random.randn(10, 4))

In [213]: df.to_csv("tmp.csv", sep="|")

In [214]: table = pd.read_csv("tmp.csv", sep="|")

In [215]: table
Out[215]: 
   Unnamed: 0         0         1         2         3
0           0 -1.294524  0.413738  0.276662 -0.472035
1           1 -0.013960 -0.362543 -0.006154 -0.923061
2           2  0.895717  0.805244 -1.206412  2.565646
3           3  1.431256  1.340309 -1.170299 -0.226169
4           4  0.410835  0.813850  0.132003 -0.827317
5           5 -0.076467 -1.187678  1.130127 -1.436737
6           6 -1.413681  1.607920  1.024180  0.569605
7           7  0.875906 -2.211372  0.974466 -2.006747
8           8 -0.410001 -0.078638  0.545952 -1.219217
9           9 -1.226825  0.769804 -1.281247 -0.727707

In [216]: with pd.read_csv("tmp.csv", sep="|", chunksize=4) as reader:
   .....:     reader
   .....:     for chunk in reader:
   .....:         print(chunk)
   .....: 
   Unnamed: 0         0         1         2         3
0           0 -1.294524  0.413738  0.276662 -0.472035
1           1 -0.013960 -0.362543 -0.006154 -0.923061
2           2  0.895717  0.805244 -1.206412  2.565646
3           3  1.431256  1.340309 -1.170299 -0.226169
   Unnamed: 0         0         1         2         3
4           4  0.410835  0.813850  0.132003 -0.827317
5           5 -0.076467 -1.187678  1.130127 -1.436737
6           6 -1.413681  1.607920  1.024180  0.569605
7           7  0.875906 -2.211372  0.974466 -2.006747
   Unnamed: 0         0         1         2         3
8           8 -0.410001 -0.078638  0.545952 -1.219217
9           9 -1.226825  0.769804 -1.281247 -0.727707

In [217]: with pd.read_csv("tmp.csv", sep="|", iterator=True) as reader:
   .....:     reader.get_chunk(5)
   .....: 

In [218]: dfj = pd.DataFrame(np.random.randn(5, 2), columns=list("AB"))

In [219]: json = dfj.to_json()

In [220]: json
Out[220]: '{"A":{"0":-0.1213062281,"1":0.6957746499,"2":0.9597255933,"3":-0.6199759194,"4":-0.7323393705},"B":{"0":-0.0978826728,"1":0.3417343559,"2":-1.1103361029,"3":0.1497483186,"4":0.6877383895}}'

In [279]: data = [
   .....:     {"id": 1, "name": {"first": "Coleen", "last": "Volk"}},
   .....:     {"name": {"given": "Mark", "family": "Regner"}},
   .....:     {"id": 2, "name": "Faye Raker"},
   .....: ]
   .....: 

In [280]: pd.json_normalize(data)
Out[280]: 
    id name.first name.last name.given name.family        name
0  1.0     Coleen      Volk        NaN         NaN         NaN
1  NaN        NaN       NaN       Mark      Regner         NaN
2  2.0        NaN       NaN        NaN         NaN  Faye Raker

In [281]: data = [
   .....:     {
   .....:         "state": "Florida",
   .....:         "shortname": "FL",
   .....:         "info": {"governor": "Rick Scott"},
   .....:         "county": [
   .....:             {"name": "Dade", "population": 12345},
   .....:             {"name": "Broward", "population": 40000},
   .....:             {"name": "Palm Beach", "population": 60000},
   .....:         ],
   .....:     },
   .....:     {
   .....:         "state": "Ohio",
   .....:         "shortname": "OH",
   .....:         "info": {"governor": "John Kasich"},
   .....:         "county": [
   .....:             {"name": "Summit", "population": 1234},
   .....:             {"name": "Cuyahoga", "population": 1337},
   .....:         ],
   .....:     },
   .....: ]
   .....: 

In [282]: pd.json_normalize(data, "county", ["state", "shortname", ["info", "governor"]])
Out[282]: 
         name  population    state shortname info.governor
0        Dade       12345  Florida        FL    Rick Scott
1     Broward       40000  Florida        FL    Rick Scott
2  Palm Beach       60000  Florida        FL    Rick Scott
3      Summit        1234     Ohio        OH   John Kasich
4    Cuyahoga        1337     Ohio        OH   John Kasich

In [283]: data = [
   .....:     {
   .....:         "CreatedBy": {"Name": "User001"},
   .....:         "Lookup": {
   .....:             "TextField": "Some text",
   .....:             "UserField": {"Id": "ID001", "Name": "Name001"},
   .....:         },
   .....:         "Image": {"a": "b"},
   .....:     }
   .....: ]
   .....: 

In [284]: pd.json_normalize(data, max_level=1)
Out[284]: 
  CreatedBy.Name Lookup.TextField                    Lookup.UserField Image.a
0        User001        Some text  {'Id': 'ID001', 'Name': 'Name001'}       b

In [285]: jsonl = """
   .....:     {"a": 1, "b": 2}
   .....:     {"a": 3, "b": 4}
   .....: """
   .....: 

In [286]: df = pd.read_json(jsonl, lines=True)

In [287]: df
Out[287]: 
   a  b
0  1  2
1  3  4

In [288]: df.to_json(orient="records", lines=True)
Out[288]: '{"a":1,"b":2}\n{"a":3,"b":4}\n'

# reader is an iterator that returns ``chunksize`` lines each iteration
In [289]: with pd.read_json(StringIO(jsonl), lines=True, chunksize=1) as reader:
   .....:     reader
   .....:     for chunk in reader:
   .....:         print(chunk)
   .....: 
Empty DataFrame
Columns: []
Index: []
   a  b
0  1  2
   a  b
1  3  4

In [290]: df = pd.DataFrame(
   .....:     {
   .....:         "A": [1, 2, 3],
   .....:         "B": ["a", "b", "c"],
   .....:         "C": pd.date_range("2016-01-01", freq="d", periods=3),
   .....:     },
   .....:     index=pd.Index(range(3), name="idx"),
   .....: )
   .....: 

In [291]: df
Out[291]: 
     A  B          C
idx                 
0    1  a 2016-01-01
1    2  b 2016-01-02
2    3  c 2016-01-03

In [292]: df.to_json(orient="table", date_format="iso")
Out[292]: '{"schema":{"fields":[{"name":"idx","type":"integer"},{"name":"A","type":"integer"},{"name":"B","type":"string"},{"name":"C","type":"datetime"}],"primaryKey":["idx"],"pandas_version":"1.4.0"},"data":[{"idx":0,"A":1,"B":"a","C":"2016-01-01T00:00:00.000"},{"idx":1,"A":2,"B":"b","C":"2016-01-02T00:00:00.000"},{"idx":2,"A":3,"B":"c","C":"2016-01-03T00:00:00.000"}]}'

In [313]: df.index.name = "index"

In [314]: df.to_json("test.json", orient="table")

In [315]: new_df = pd.read_json("test.json", orient="table")

In [316]: print(new_df.index.name)
None

In [320]: "https://www.fdic.gov/resources/resolutions/bank-failures/failed-bank-list"
In [321]: pd.read_html(url)
Out[321]:
[                         Bank NameBank           CityCity StateSt  ...              Acquiring InstitutionAI Closing DateClosing FundFund
 0                    Almena State Bank             Almena      KS  ...                          Equity Bank    October 23, 2020    10538
 1           First City Bank of Florida  Fort Walton Beach      FL  ...            United Fidelity Bank, fsb    October 16, 2020    10537
 2                 The First State Bank      Barboursville      WV  ...                       MVB Bank, Inc.       April 3, 2020    10536
 3                   Ericson State Bank            Ericson      NE  ...           Farmers and Merchants Bank   February 14, 2020    10535
 4     City National Bank of New Jersey             Newark      NJ  ...                      Industrial Bank    November 1, 2019    10534
 ..                                 ...                ...     ...  ...                                  ...                 ...      ...
 558                 Superior Bank, FSB           Hinsdale      IL  ...                Superior Federal, FSB       July 27, 2001     6004
 559                Malta National Bank              Malta      OH  ...                    North Valley Bank         May 3, 2001     4648
 560    First Alliance Bank & Trust Co.         Manchester      NH  ...  Southern New Hampshire Bank & Trust    February 2, 2001     4647
 561  National State Bank of Metropolis         Metropolis      IL  ...              Banterra Bank of Marion   December 14, 2000     4646
 562                   Bank of Honolulu           Honolulu      HI  ...                   Bank of the Orient    October 13, 2000     4645

 [563 rows x 7 columns]]

In [317]: html_str = """
   .....:          <table>
   .....:              <tr>
   .....:                  <th>A</th>
   .....:                  <th colspan="1">B</th>
   .....:                  <th rowspan="1">C</th>
   .....:              </tr>
   .....:              <tr>
   .....:                  <td>a</td>
   .....:                  <td>b</td>
   .....:                  <td>c</td>
   .....:              </tr>
   .....:          </table>
   .....:      """
   .....: 

In [318]: with open("tmp.html", "w") as f:
   .....:     f.write(html_str)
   .....: 

In [319]: df = pd.read_html("tmp.html")

In [320]: df[0]
Out[320]: 
   A  B  C
0  a  b  c

In [321]: dfs = pd.read_html(StringIO(html_str))

In [322]: dfs[0]
Out[322]: 
   A  B  C
0  a  b  c

In [323]: html_table = """
   .....: <table>
   .....:   <tr>
   .....:     <th>GitHub</th>
   .....:   </tr>
   .....:   <tr>
   .....:     <td><a href="https://github.com/pandas-dev/pandas">pandas</a></td>
   .....:   </tr>
   .....: </table>
   .....: """
   .....: 

In [324]: df = pd.read_html(
   .....:     html_table,
   .....:     extract_links="all"
   .....: )[0]
   .....: 

In [325]: df
Out[325]: 
                                   (GitHub, None)
0  (pandas, https://github.com/pandas-dev/pandas)

In [326]: df[("GitHub", None)]
Out[326]: 
0    (pandas, https://github.com/pandas-dev/pandas)
Name: (GitHub, None), dtype: object

In [327]: df[("GitHub", None)].str[1]
Out[327]: 
0    https://github.com/pandas-dev/pandas
Name: (GitHub, None), dtype: object

In [328]: from IPython.display import display, HTML

In [329]: df = pd.DataFrame(np.random.randn(2, 2))

In [330]: df
Out[330]: 
          0         1
0  0.070319  1.773907
1  0.253908  0.414581

In [331]: html = df.to_html()

In [332]: print(html)  # raw html
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>0</th>
      <th>1</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0.070319</td>
      <td>1.773907</td>
    </tr>
    <tr>
      <th>1</th>
      <td>0.253908</td>
      <td>0.414581</td>
    </tr>
  </tbody>
</table>

In [333]: display(HTML(html))
<IPython.core.display.HTML object>

In [334]: html = df.to_html(columns=[0])

In [335]: print(html)
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>0</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0.070319</td>
    </tr>
    <tr>
      <th>1</th>
      <td>0.253908</td>
    </tr>
  </tbody>
</table>

In [336]: display(HTML(html))
<IPython.core.display.HTML object>

In [337]: html = df.to_html(float_format="{0:.10f}".format)

In [338]: print(html)
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>0</th>
      <th>1</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0.0703192665</td>
      <td>1.7739074228</td>
    </tr>
    <tr>
      <th>1</th>
      <td>0.2539083433</td>
      <td>0.4145805920</td>
    </tr>
  </tbody>
</table>

In [339]: display(HTML(html))
<IPython.core.display.HTML object>

In [340]: html = df.to_html(bold_rows=False)

In [341]: print(html)
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>0</th>
      <th>1</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td>0</td>
      <td>0.070319</td>
      <td>1.773907</td>
    </tr>
    <tr>
      <td>1</td>
      <td>0.253908</td>
      <td>0.414581</td>
    </tr>
  </tbody>
</table>

In [342]: display(HTML(html))
<IPython.core.display.HTML object>

In [343]: print(df.to_html(classes=["awesome_table_class", "even_more_awesome_class"]))
<table border="1" class="dataframe awesome_table_class even_more_awesome_class">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>0</th>
      <th>1</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>0.070319</td>
      <td>1.773907</td>
    </tr>
    <tr>
      <th>1</th>
      <td>0.253908</td>
      <td>0.414581</td>
    </tr>
  </tbody>
</table>

In [344]: url_df = pd.DataFrame(
   .....:     {
   .....:         "name": ["Python", "pandas"],
   .....:         "url": ["https://www.python.org/", "https://pandas.pydata.org"],
   .....:     }
   .....: )
   .....: 

In [345]: html = url_df.to_html(render_links=True)

In [346]: print(html)
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>name</th>
      <th>url</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Python</td>
      <td><a href="https://www.python.org/" target="_blank">https://www.python.org/</a></td>
    </tr>
    <tr>
      <th>1</th>
      <td>pandas</td>
      <td><a href="https://pandas.pydata.org" target="_blank">https://pandas.pydata.org</a></td>
    </tr>
  </tbody>
</table>

In [347]: display(HTML(html))
<IPython.core.display.HTML object>

In [348]: df = pd.DataFrame({"a": list("&<>"), "b": np.random.randn(3)})

In [349]: html = df.to_html()

In [350]: print(html)
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>a</th>
      <th>b</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>&amp;</td>
      <td>0.842321</td>
    </tr>
    <tr>
      <th>1</th>
      <td>&lt;</td>
      <td>0.211337</td>
    </tr>
    <tr>
      <th>2</th>
      <td>&gt;</td>
      <td>-1.055427</td>
    </tr>
  </tbody>
</table>

In [351]: display(HTML(html))
<IPython.core.display.HTML object>

In [352]: html = df.to_html(escape=False)

In [353]: print(html)
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>a</th>
      <th>b</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>&</td>
      <td>0.842321</td>
    </tr>
    <tr>
      <th>1</th>
      <td><</td>
      <td>0.211337</td>
    </tr>
    <tr>
      <th>2</th>
      <td>></td>
      <td>-1.055427</td>
    </tr>
  </tbody>
</table>

In [354]: display(HTML(html))
<IPython.core.display.HTML object>

In [355]: df = pd.DataFrame([[1, 2], [3, 4]], index=["a", "b"], columns=["c", "d"])

In [356]: print(df.style.to_latex())
\begin{tabular}{lrr}
 & c & d \\
a & 1 & 2 \\
b & 3 & 4 \\
\end{tabular}

In [357]: print(df.style.format("€ {}").to_latex())
\begin{tabular}{lrr}
 & c & d \\
a & € 1 & € 2 \\
b & € 3 & € 4 \\
\end{tabular}

In [358]: xml = """<?xml version="1.0" encoding="UTF-8"?>
   .....: <bookstore>
   .....:   <book category="cooking">
   .....:     <title lang="en">Everyday Italian</title>
   .....:     <author>Giada De Laurentiis</author>
   .....:     <year>2005</year>
   .....:     <price>30.00</price>
   .....:   </book>
   .....:   <book category="children">
   .....:     <title lang="en">Harry Potter</title>
   .....:     <author>J K. Rowling</author>
   .....:     <year>2005</year>
   .....:     <price>29.99</price>
   .....:   </book>
   .....:   <book category="web">
   .....:     <title lang="en">Learning XML</title>
   .....:     <author>Erik T. Ray</author>
   .....:     <year>2003</year>
   .....:     <price>39.95</price>
   .....:   </book>
   .....: </bookstore>"""
   .....: 

In [359]: df = pd.read_xml(xml)

In [360]: df
Out[360]: 
   category             title               author  year  price
0   cooking  Everyday Italian  Giada De Laurentiis  2005  30.00
1  children      Harry Potter         J K. Rowling  2005  29.99
2       web      Learning XML          Erik T. Ray  2003  39.95

In [361]: df = pd.read_xml("https://www.w3schools.com/xml/books.xml")

In [362]: df
Out[362]: 
   category              title                  author  year  price      cover
0   cooking   Everyday Italian     Giada De Laurentiis  2005  30.00       None
1  children       Harry Potter            J K. Rowling  2005  29.99       None
2       web  XQuery Kick Start  Vaidyanathan Nagarajan  2003  49.99       None
3       web       Learning XML             Erik T. Ray  2003  39.95  paperback

In [363]: file_path = "books.xml"

In [364]: with open(file_path, "w") as f:
   .....:     f.write(xml)
   .....: 

In [365]: with open(file_path, "r") as f:
   .....:     df = pd.read_xml(f.read())
   .....: 

In [366]: df
Out[366]: 
   category             title               author  year  price
0   cooking  Everyday Italian  Giada De Laurentiis  2005  30.00
1  children      Harry Potter         J K. Rowling  2005  29.99
2       web      Learning XML          Erik T. Ray  2003  39.95

In [367]: with open(file_path, "r") as f:
   .....:     sio = StringIO(f.read())
   .....: 

In [368]: df = pd.read_xml(sio)

In [369]: df
Out[369]: 
   category             title               author  year  price
0   cooking  Everyday Italian  Giada De Laurentiis  2005  30.00
1  children      Harry Potter         J K. Rowling  2005  29.99
2       web      Learning XML          Erik T. Ray  2003  39.95

In [370]: with open(file_path, "rb") as f:
   .....:     bio = BytesIO(f.read())
   .....: 

In [371]: df = pd.read_xml(bio)

In [372]: df
Out[372]: 
   category             title               author  year  price
0   cooking  Everyday Italian  Giada De Laurentiis  2005  30.00
1  children      Harry Potter         J K. Rowling  2005  29.99
2       web      Learning XML          Erik T. Ray  2003  39.95

In [373]: df = pd.read_xml(
   .....:     "s3://pmc-oa-opendata/oa_comm/xml/all/PMC1236943.xml",
   .....:     xpath=".//journal-meta",
   .....: )
   .....: 

In [374]: df
Out[374]: 
              journal-id              journal-title       issn  publisher
0  Cardiovasc Ultrasound  Cardiovascular Ultrasound  1476-7120        NaN

In [375]: df = pd.read_xml(file_path, xpath="//book[year=2005]")

In [376]: df
Out[376]: 
   category             title               author  year  price
0   cooking  Everyday Italian  Giada De Laurentiis  2005  30.00
1  children      Harry Potter         J K. Rowling  2005  29.99

In [377]: df = pd.read_xml(file_path, elems_only=True)

In [378]: df
Out[378]: 
              title               author  year  price
0  Everyday Italian  Giada De Laurentiis  2005  30.00
1      Harry Potter         J K. Rowling  2005  29.99
2      Learning XML          Erik T. Ray  2003  39.95

In [379]: df = pd.read_xml(file_path, attrs_only=True)

In [380]: df
Out[380]: 
   category
0   cooking
1  children
2       web

In [381]: xml = """<?xml version='1.0' encoding='utf-8'?>
   .....: <doc:data xmlns:doc="https://example.com">
   .....:   <doc:row>
   .....:     <doc:shape>square</doc:shape>
   .....:     <doc:degrees>360</doc:degrees>
   .....:     <doc:sides>4.0</doc:sides>
   .....:   </doc:row>
   .....:   <doc:row>
   .....:     <doc:shape>circle</doc:shape>
   .....:     <doc:degrees>360</doc:degrees>
   .....:     <doc:sides/>
   .....:   </doc:row>
   .....:   <doc:row>
   .....:     <doc:shape>triangle</doc:shape>
   .....:     <doc:degrees>180</doc:degrees>
   .....:     <doc:sides>3.0</doc:sides>
   .....:   </doc:row>
   .....: </doc:data>"""
   .....: 

In [382]: df = pd.read_xml(xml,
   .....:                  xpath="//doc:row",
   .....:                  namespaces={"doc": "https://example.com"})
   .....: 

In [383]: df
Out[383]: 
      shape  degrees  sides
0    square      360    4.0
1    circle      360    NaN
2  triangle      180    3.0

In [384]: xml = """<?xml version='1.0' encoding='utf-8'?>
   .....: <data xmlns="https://example.com">
   .....:  <row>
   .....:    <shape>square</shape>
   .....:    <degrees>360</degrees>
   .....:    <sides>4.0</sides>
   .....:  </row>
   .....:  <row>
   .....:    <shape>circle</shape>
   .....:    <degrees>360</degrees>
   .....:    <sides/>
   .....:  </row>
   .....:  <row>
   .....:    <shape>triangle</shape>
   .....:    <degrees>180</degrees>
   .....:    <sides>3.0</sides>
   .....:  </row>
   .....: </data>"""
   .....: 

In [385]: df = pd.read_xml(xml,
   .....:                  xpath="//pandas:row",
   .....:                  namespaces={"pandas": "https://example.com"})
   .....: 

In [386]: df
Out[386]: 
      shape  degrees  sides
0    square      360    4.0
1    circle      360    NaN
2  triangle      180    3.0

In [387]: xml = """<?xml version='1.0' encoding='utf-8'?>
   .....:  <response>
   .....:   <row>
   .....:     <station id="40850" name="Library"/>
   .....:     <month>2020-09-01T00:00:00</month>
   .....:     <rides>
   .....:       <avg_weekday_rides>864.2</avg_weekday_rides>
   .....:       <avg_saturday_rides>534</avg_saturday_rides>
   .....:       <avg_sunday_holiday_rides>417.2</avg_sunday_holiday_rides>
   .....:     </rides>
   .....:   </row>
   .....:   <row>
   .....:     <station id="41700" name="Washington/Wabash"/>
   .....:     <month>2020-09-01T00:00:00</month>
   .....:     <rides>
   .....:       <avg_weekday_rides>2707.4</avg_weekday_rides>
   .....:       <avg_saturday_rides>1909.8</avg_saturday_rides>
   .....:       <avg_sunday_holiday_rides>1438.6</avg_sunday_holiday_rides>
   .....:     </rides>
   .....:   </row>
   .....:   <row>
   .....:     <station id="40380" name="Clark/Lake"/>
   .....:     <month>2020-09-01T00:00:00</month>
   .....:     <rides>
   .....:       <avg_weekday_rides>2949.6</avg_weekday_rides>
   .....:       <avg_saturday_rides>1657</avg_saturday_rides>
   .....:       <avg_sunday_holiday_rides>1453.8</avg_sunday_holiday_rides>
   .....:     </rides>
   .....:   </row>
   .....:  </response>"""
   .....: 

In [388]: xsl = """<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
   .....:    <xsl:output method="xml" omit-xml-declaration="no" indent="yes"/>
   .....:    <xsl:strip-space elements="*"/>
   .....:    <xsl:template match="/response">
   .....:       <xsl:copy>
   .....:         <xsl:apply-templates select="row"/>
   .....:       </xsl:copy>
   .....:    </xsl:template>
   .....:    <xsl:template match="row">
   .....:       <xsl:copy>
   .....:         <station_id><xsl:value-of select="station/@id"/></station_id>
   .....:         <station_name><xsl:value-of select="station/@name"/></station_name>
   .....:         <xsl:copy-of select="month|rides/*"/>
   .....:       </xsl:copy>
   .....:    </xsl:template>
   .....:  </xsl:stylesheet>"""
   .....: 

In [389]: output = """<?xml version='1.0' encoding='utf-8'?>
   .....:  <response>
   .....:    <row>
   .....:       <station_id>40850</station_id>
   .....:       <station_name>Library</station_name>
   .....:       <month>2020-09-01T00:00:00</month>
   .....:       <avg_weekday_rides>864.2</avg_weekday_rides>
   .....:       <avg_saturday_rides>534</avg_saturday_rides>
   .....:       <avg_sunday_holiday_rides>417.2</avg_sunday_holiday_rides>
   .....:    </row>
   .....:    <row>
   .....:       <station_id>41700</station_id>
   .....:       <station_name>Washington/Wabash</station_name>
   .....:       <month>2020-09-01T00:00:00</month>
   .....:       <avg_weekday_rides>2707.4</avg_weekday_rides>
   .....:       <avg_saturday_rides>1909.8</avg_saturday_rides>
   .....:       <avg_sunday_holiday_rides>1438.6</avg_sunday_holiday_rides>
   .....:    </row>
   .....:    <row>
   .....:       <station_id>40380</station_id>
   .....:       <station_name>Clark/Lake</station_name>
   .....:       <month>2020-09-01T00:00:00</month>
   .....:       <avg_weekday_rides>2949.6</avg_weekday_rides>
   .....:       <avg_saturday_rides>1657</avg_saturday_rides>
   .....:       <avg_sunday_holiday_rides>1453.8</avg_sunday_holiday_rides>
   .....:    </row>
   .....:  </response>"""
   .....: 

In [390]: df = pd.read_xml(xml, stylesheet=xsl)

In [391]: df
Out[391]: 
   station_id       station_name  ... avg_saturday_rides  avg_sunday_holiday_rides
0       40850            Library  ...              534.0                     417.2
1       41700  Washington/Wabash  ...             1909.8                    1438.6
2       40380         Clark/Lake  ...             1657.0                    1453.8

[3 rows x 6 columns]

In [1]: df = pd.read_xml(
...         "/path/to/downloaded/enwikisource-latest-pages-articles.xml",
...         iterparse = {"page": ["title", "ns", "id"]}
...     )
...     df
Out[2]:
                                                     title   ns        id
0                                       Gettysburg Address    0     21450
1                                                Main Page    0     42950
2                            Declaration by United Nations    0      8435
3             Constitution of the United States of America    0      8435
4                     Declaration of Independence (Israel)    0     17858
...                                                    ...  ...       ...
3578760               Page:Black cat 1897 07 v2 n10.pdf/17  104    219649
3578761               Page:Black cat 1897 07 v2 n10.pdf/43  104    219649
3578762               Page:Black cat 1897 07 v2 n10.pdf/44  104    219649
3578763      The History of Tom Jones, a Foundling/Book IX    0  12084291
3578764  Page:Shakespeare of Stratford (1926) Yale.djvu/91  104     21450

[3578765 rows x 3 columns]

In [392]: geom_df = pd.DataFrame(
   .....:     {
   .....:         "shape": ["square", "circle", "triangle"],
   .....:         "degrees": [360, 360, 180],
   .....:         "sides": [4, np.nan, 3],
   .....:     }
   .....: )
   .....: 

In [393]: print(geom_df.to_xml())
<?xml version='1.0' encoding='utf-8'?>
<data>
  <row>
    <index>0</index>
    <shape>square</shape>
    <degrees>360</degrees>
    <sides>4.0</sides>
  </row>
  <row>
    <index>1</index>
    <shape>circle</shape>
    <degrees>360</degrees>
    <sides/>
  </row>
  <row>
    <index>2</index>
    <shape>triangle</shape>
    <degrees>180</degrees>
    <sides>3.0</sides>
  </row>
</data>

In [394]: print(geom_df.to_xml(root_name="geometry", row_name="objects"))
<?xml version='1.0' encoding='utf-8'?>
<geometry>
  <objects>
    <index>0</index>
    <shape>square</shape>
    <degrees>360</degrees>
    <sides>4.0</sides>
  </objects>
  <objects>
    <index>1</index>
    <shape>circle</shape>
    <degrees>360</degrees>
    <sides/>
  </objects>
  <objects>
    <index>2</index>
    <shape>triangle</shape>
    <degrees>180</degrees>
    <sides>3.0</sides>
  </objects>
</geometry>

In [395]: print(geom_df.to_xml(attr_cols=geom_df.columns.tolist()))
<?xml version='1.0' encoding='utf-8'?>
<data>
  <row index="0" shape="square" degrees="360" sides="4.0"/>
  <row index="1" shape="circle" degrees="360"/>
  <row index="2" shape="triangle" degrees="180" sides="3.0"/>
</data>

In [396]: print(
   .....:     geom_df.to_xml(
   .....:         index=False,
   .....:         attr_cols=['shape'],
   .....:         elem_cols=['degrees', 'sides'])
   .....: )
   .....: 
<?xml version='1.0' encoding='utf-8'?>
<data>
  <row shape="square">
    <degrees>360</degrees>
    <sides>4.0</sides>
  </row>
  <row shape="circle">
    <degrees>360</degrees>
    <sides/>
  </row>
  <row shape="triangle">
    <degrees>180</degrees>
    <sides>3.0</sides>
  </row>
</data>

In [397]: ext_geom_df = pd.DataFrame(
   .....:     {
   .....:         "type": ["polygon", "other", "polygon"],
   .....:         "shape": ["square", "circle", "triangle"],
   .....:         "degrees": [360, 360, 180],
   .....:         "sides": [4, np.nan, 3],
   .....:     }
   .....: )
   .....: 

In [398]: pvt_df = ext_geom_df.pivot_table(index='shape',
   .....:                                  columns='type',
   .....:                                  values=['degrees', 'sides'],
   .....:                                  aggfunc='sum')
   .....: 

In [399]: pvt_df
Out[399]: 
         degrees         sides        
type       other polygon other polygon
shape                                 
circle     360.0     NaN   0.0     NaN
square       NaN   360.0   NaN     4.0
triangle     NaN   180.0   NaN     3.0

In [400]: print(pvt_df.to_xml())
<?xml version='1.0' encoding='utf-8'?>
<data>
  <row>
    <shape>circle</shape>
    <degrees_other>360.0</degrees_other>
    <degrees_polygon/>
    <sides_other>0.0</sides_other>
    <sides_polygon/>
  </row>
  <row>
    <shape>square</shape>
    <degrees_other/>
    <degrees_polygon>360.0</degrees_polygon>
    <sides_other/>
    <sides_polygon>4.0</sides_polygon>
  </row>
  <row>
    <shape>triangle</shape>
    <degrees_other/>
    <degrees_polygon>180.0</degrees_polygon>
    <sides_other/>
    <sides_polygon>3.0</sides_polygon>
  </row>
</data>

In [401]: print(geom_df.to_xml(namespaces={"": "https://example.com"}))
<?xml version='1.0' encoding='utf-8'?>
<data xmlns="https://example.com">
  <row>
    <index>0</index>
    <shape>square</shape>
    <degrees>360</degrees>
    <sides>4.0</sides>
  </row>
  <row>
    <index>1</index>
    <shape>circle</shape>
    <degrees>360</degrees>
    <sides/>
  </row>
  <row>
    <index>2</index>
    <shape>triangle</shape>
    <degrees>180</degrees>
    <sides>3.0</sides>
  </row>
</data>

In [402]: print(
   .....:     geom_df.to_xml(namespaces={"doc": "https://example.com"},
   .....:                    prefix="doc")
   .....: )
   .....: 
<?xml version='1.0' encoding='utf-8'?>
<doc:data xmlns:doc="https://example.com">
  <doc:row>
    <doc:index>0</doc:index>
    <doc:shape>square</doc:shape>
    <doc:degrees>360</doc:degrees>
    <doc:sides>4.0</doc:sides>
  </doc:row>
  <doc:row>
    <doc:index>1</doc:index>
    <doc:shape>circle</doc:shape>
    <doc:degrees>360</doc:degrees>
    <doc:sides/>
  </doc:row>
  <doc:row>
    <doc:index>2</doc:index>
    <doc:shape>triangle</doc:shape>
    <doc:degrees>180</doc:degrees>
    <doc:sides>3.0</doc:sides>
  </doc:row>
</doc:data>

In [403]: print(
   .....:     geom_df.to_xml(xml_declaration=False,
   .....:                    pretty_print=False)
   .....: )
   .....: 
<data><row><index>0</index><shape>square</shape><degrees>360</degrees><sides>4.0</sides></row><row><index>1</index><shape>circle</shape><degrees>360</degrees><sides/></row><row><index>2</index><shape>triangle</shape><degrees>180</degrees><sides>3.0</sides></row></data>

In [404]: xsl = """<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
   .....:    <xsl:output method="xml" omit-xml-declaration="no" indent="yes"/>
   .....:    <xsl:strip-space elements="*"/>
   .....:    <xsl:template match="/data">
   .....:      <geometry>
   .....:        <xsl:apply-templates select="row"/>
   .....:      </geometry>
   .....:    </xsl:template>
   .....:    <xsl:template match="row">
   .....:      <object index="{index}">
   .....:        <xsl:if test="shape!='circle'">
   .....:            <xsl:attribute name="type">polygon</xsl:attribute>
   .....:        </xsl:if>
   .....:        <xsl:copy-of select="shape"/>
   .....:        <property>
   .....:            <xsl:copy-of select="degrees|sides"/>
   .....:        </property>
   .....:      </object>
   .....:    </xsl:template>
   .....:  </xsl:stylesheet>"""
   .....: 

In [405]: print(geom_df.to_xml(stylesheet=xsl))
<?xml version="1.0"?>
<geometry>
  <object index="0" type="polygon">
    <shape>square</shape>
    <property>
      <degrees>360</degrees>
      <sides>4.0</sides>
    </property>
  </object>
  <object index="1">
    <shape>circle</shape>
    <property>
      <degrees>360</degrees>
      <sides/>
    </property>
  </object>
  <object index="2" type="polygon">
    <shape>triangle</shape>
    <property>
      <degrees>180</degrees>
      <sides>3.0</sides>
    </property>
  </object>
</geometry>

In [421]: df = pd.DataFrame(
   .....:     {
   .....:         "A": np.random.randn(1000),
   .....:         "B": "foo",
   .....:         "C": pd.date_range("20130101", periods=1000, freq="s"),
   .....:     }
   .....: )
   .....: 

In [422]: df
Out[422]: 
            A    B                   C
0   -0.828876  foo 2013-01-01 00:00:00
1   -0.110383  foo 2013-01-01 00:00:01
2    2.357598  foo 2013-01-01 00:00:02
3   -1.620073  foo 2013-01-01 00:00:03
4    0.440903  foo 2013-01-01 00:00:04
..        ...  ...                 ...
995 -1.177365  foo 2013-01-01 00:16:35
996  1.236988  foo 2013-01-01 00:16:36
997  0.743946  foo 2013-01-01 00:16:37
998 -0.533097  foo 2013-01-01 00:16:38
999 -0.140850  foo 2013-01-01 00:16:39

[1000 rows x 3 columns]

In [423]: df.to_pickle("data.pkl.compress", compression="gzip")

In [424]: rt = pd.read_pickle("data.pkl.compress", compression="gzip")

In [425]: rt
Out[425]: 
            A    B                   C
0   -0.828876  foo 2013-01-01 00:00:00
1   -0.110383  foo 2013-01-01 00:00:01
2    2.357598  foo 2013-01-01 00:00:02
3   -1.620073  foo 2013-01-01 00:00:03
4    0.440903  foo 2013-01-01 00:00:04
..        ...  ...                 ...
995 -1.177365  foo 2013-01-01 00:16:35
996  1.236988  foo 2013-01-01 00:16:36
997  0.743946  foo 2013-01-01 00:16:37
998 -0.533097  foo 2013-01-01 00:16:38
999 -0.140850  foo 2013-01-01 00:16:39

[1000 rows x 3 columns]

In [426]: df.to_pickle("data.pkl.xz", compression="infer")

In [427]: rt = pd.read_pickle("data.pkl.xz", compression="infer")

In [428]: rt
Out[428]: 
            A    B                   C
0   -0.828876  foo 2013-01-01 00:00:00
1   -0.110383  foo 2013-01-01 00:00:01
2    2.357598  foo 2013-01-01 00:00:02
3   -1.620073  foo 2013-01-01 00:00:03
4    0.440903  foo 2013-01-01 00:00:04
..        ...  ...                 ...
995 -1.177365  foo 2013-01-01 00:16:35
996  1.236988  foo 2013-01-01 00:16:36
997  0.743946  foo 2013-01-01 00:16:37
998 -0.533097  foo 2013-01-01 00:16:38
999 -0.140850  foo 2013-01-01 00:16:39

[1000 rows x 3 columns]

In [429]: df.to_pickle("data.pkl.gz")

In [430]: rt = pd.read_pickle("data.pkl.gz")

In [431]: rt
Out[431]: 
            A    B                   C
0   -0.828876  foo 2013-01-01 00:00:00
1   -0.110383  foo 2013-01-01 00:00:01
2    2.357598  foo 2013-01-01 00:00:02
3   -1.620073  foo 2013-01-01 00:00:03
4    0.440903  foo 2013-01-01 00:00:04
..        ...  ...                 ...
995 -1.177365  foo 2013-01-01 00:16:35
996  1.236988  foo 2013-01-01 00:16:36
997  0.743946  foo 2013-01-01 00:16:37
998 -0.533097  foo 2013-01-01 00:16:38
999 -0.140850  foo 2013-01-01 00:16:39

[1000 rows x 3 columns]

In [432]: df["A"].to_pickle("s1.pkl.bz2")

In [433]: rt = pd.read_pickle("s1.pkl.bz2")

In [434]: rt
Out[434]: 
0     -0.828876
1     -0.110383
2      2.357598
3     -1.620073
4      0.440903
         ...   
995   -1.177365
996    1.236988
997    0.743946
998   -0.533097
999   -0.140850
Name: A, Length: 1000, dtype: float64

In [435]: df.to_pickle("data.pkl.gz", compression={"method": "gzip", "compresslevel": 1})

In [452]: df_tl = pd.DataFrame({"A": list(range(5)), "B": list(range(5))})

In [453]: df_tl.to_hdf("store_tl.h5", "table", append=True)

In [454]: pd.read_hdf("store_tl.h5", "table", where=["index>2"])
Out[454]: 
   A  B
3  3  3
4  4  4

In [455]: df_with_missing = pd.DataFrame(
   .....:     {
   .....:         "col1": [0, np.nan, 2],
   .....:         "col2": [1, np.nan, np.nan],
   .....:     }
   .....: )
   .....: 

In [456]: df_with_missing
Out[456]: 
   col1  col2
0   0.0   1.0
1   NaN   NaN
2   2.0   NaN

In [457]: df_with_missing.to_hdf("file.h5", "df_with_missing", format="table", mode="w")

In [458]: pd.read_hdf("file.h5", "df_with_missing")
Out[458]: 
   col1  col2
0   0.0   1.0
1   NaN   NaN
2   2.0   NaN

In [459]: df_with_missing.to_hdf(
   .....:     "file.h5", "df_with_missing", format="table", mode="w", dropna=True
   .....: )
   .....: 

In [460]: pd.read_hdf("file.h5", "df_with_missing")
Out[460]: 
   col1  col2
0   0.0   1.0
2   2.0   NaN

In [461]: store = pd.HDFStore("store.h5")

In [462]: df1 = df[0:4]

In [463]: df2 = df[4:]

# append data (creates a table automatically)
In [464]: store.append("df", df1)

In [465]: store.append("df", df2)

In [466]: store
Out[466]: 
<class 'pandas.io.pytables.HDFStore'>
File path: store.h5

# select the entire object
In [467]: store.select("df")
Out[467]: 
                   A         B         C
2000-01-01 -0.398501 -0.677311 -0.874991
2000-01-02 -1.167564 -0.593353  0.146262
2000-01-03 -0.131959  0.089012  0.667450
2000-01-04  0.169405 -1.358046 -0.105563
2000-01-05  0.492195  0.076693  0.213685
2000-01-06 -0.285283 -1.210529 -1.408386
2000-01-07  0.941577 -0.342447  0.222031
2000-01-08  0.052607  2.093214  1.064908

# the type of stored data
In [468]: store.root.df._v_attrs.pandas_type
Out[468]: 'frame_table'

In [469]: store.put("foo/bar/bah", df)

In [470]: store.append("food/orange", df)

In [471]: store.append("food/apple", df)

In [472]: store
Out[472]: 
<class 'pandas.io.pytables.HDFStore'>
File path: store.h5

# a list of keys are returned
In [473]: store.keys()
Out[473]: ['/df', '/food/apple', '/food/orange', '/foo/bar/bah']

# remove all nodes under this level
In [474]: store.remove("food")

In [475]: store
Out[475]: 
<class 'pandas.io.pytables.HDFStore'>
File path: store.h5

In [476]: for (path, subgroups, subkeys) in store.walk():
   .....:     for subgroup in subgroups:
   .....:         print("GROUP: {}/{}".format(path, subgroup))
   .....:     for subkey in subkeys:
   .....:         key = "/".join([path, subkey])
   .....:         print("KEY: {}".format(key))
   .....:         print(store.get(key))
   .....: 
GROUP: /foo
KEY: /df
                   A         B         C
2000-01-01 -0.398501 -0.677311 -0.874991
2000-01-02 -1.167564 -0.593353  0.146262
2000-01-03 -0.131959  0.089012  0.667450
2000-01-04  0.169405 -1.358046 -0.105563
2000-01-05  0.492195  0.076693  0.213685
2000-01-06 -0.285283 -1.210529 -1.408386
2000-01-07  0.941577 -0.342447  0.222031
2000-01-08  0.052607  2.093214  1.064908
GROUP: /foo/bar
KEY: /foo/bar/bah
                   A         B         C
2000-01-01 -0.398501 -0.677311 -0.874991
2000-01-02 -1.167564 -0.593353  0.146262
2000-01-03 -0.131959  0.089012  0.667450
2000-01-04  0.169405 -1.358046 -0.105563
2000-01-05  0.492195  0.076693  0.213685
2000-01-06 -0.285283 -1.210529 -1.408386
2000-01-07  0.941577 -0.342447  0.222031
2000-01-08  0.052607  2.093214  1.064908

In [584]: df_for_r = pd.DataFrame(
   .....:     {
   .....:         "first": np.random.rand(100),
   .....:         "second": np.random.rand(100),
   .....:         "class": np.random.randint(0, 2, (100,)),
   .....:     },
   .....:     index=range(100),
   .....: )
   .....: 

In [585]: df_for_r.head()
Out[585]: 
      first    second  class
0  0.013480  0.504941      0
1  0.690984  0.898188      1
2  0.510113  0.618748      1
3  0.357698  0.004972      0
4  0.451658  0.012065      1

In [586]: store_export = pd.HDFStore("export.h5")

In [587]: store_export.append("df_for_r", df_for_r, data_columns=df_dc.columns)

In [588]: store_export
Out[588]: 
<class 'pandas.io.pytables.HDFStore'>
File path: export.h5

In [607]: df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})

In [608]: df.to_parquet("test.parquet", engine="pyarrow")

In [609]: df.to_parquet("test.parquet", index=False)

In [610]: df = pd.DataFrame({"a": [0, 0, 1, 1], "b": [0, 1, 0, 1]})

In [611]: df.to_parquet(path="test", engine="pyarrow", partition_cols=["a"], compression=None)

In [622]: import datetime

In [623]: c = ["id", "Date", "Col_1", "Col_2", "Col_3"]

In [624]: d = [
   .....:     (26, datetime.datetime(2010, 10, 18), "X", 27.5, True),
   .....:     (42, datetime.datetime(2010, 10, 19), "Y", -12.5, False),
   .....:     (63, datetime.datetime(2010, 10, 20), "Z", 5.73, True),
   .....: ]
   .....: 

In [625]: data = pd.DataFrame(d, columns=c)

In [626]: data
Out[626]: 
   id       Date Col_1  Col_2  Col_3
0  26 2010-10-18     X  27.50   True
1  42 2010-10-19     Y -12.50  False
2  63 2010-10-20     Z   5.73   True

In [627]: data.to_sql("data", engine)
Out[627]: 3

In [628]: data.to_sql("data_chunked", engine, chunksize=1000)
Out[628]: 3

In [631]: pd.read_sql_table("data", engine)
Out[631]: 
   index  id       Date Col_1  Col_2  Col_3
0      0  26 2010-10-18     X  27.50   True
1      1  42 2010-10-19     Y -12.50  False
2      2  63 2010-10-20     Z   5.73   True

In [632]: pd.read_sql_table("data", engine, index_col="id")
Out[632]: 
    index       Date Col_1  Col_2  Col_3
id                                      
26      0 2010-10-18     X  27.50   True
42      1 2010-10-19     Y -12.50  False
63      2 2010-10-20     Z   5.73   True

In [633]: pd.read_sql_table("data", engine, columns=["Col_1", "Col_2"])
Out[633]: 
  Col_1  Col_2
0     X  27.50
1     Y -12.50
2     Z   5.73

In [634]: pd.read_sql_table("data", engine, parse_dates=["Date"])
Out[634]: 
   index  id       Date Col_1  Col_2  Col_3
0      0  26 2010-10-18     X  27.50   True
1      1  42 2010-10-19     Y -12.50  False
2      2  63 2010-10-20     Z   5.73   True

In [635]: pd.read_sql_query("SELECT * FROM data", engine)
Out[635]: 
   index  id                        Date Col_1  Col_2  Col_3
0      0  26  2010-10-18 00:00:00.000000     X  27.50      1
1      1  42  2010-10-19 00:00:00.000000     Y -12.50      0
2      2  63  2010-10-20 00:00:00.000000     Z   5.73      1

In [636]: pd.read_sql_query("SELECT id, Col_1, Col_2 FROM data WHERE id = 42;", engine)
Out[636]: 
   id Col_1  Col_2
0  42     Y  -12.5

In [637]: df = pd.DataFrame(np.random.randn(20, 3), columns=list("abc"))

In [638]: df.to_sql("data_chunks", engine, index=False)
Out[638]: 20

In [639]: for chunk in pd.read_sql_query("SELECT * FROM data_chunks", engine, chunksize=5):
   .....:     print(chunk)
   .....: 
          a         b         c
0  0.070470  0.901320  0.937577
1  0.295770  1.420548 -0.005283
2 -1.518598 -0.730065  0.226497
3 -2.061465  0.632115  0.853619
4  2.719155  0.139018  0.214557
          a         b         c
0 -1.538924 -0.366973 -0.748801
1 -0.478137 -1.559153 -3.097759
2 -2.320335 -0.221090  0.119763
3  0.608228  1.064810 -0.780506
4 -2.736887  0.143539  1.170191
          a         b         c
0 -1.573076  0.075792 -1.722223
1 -0.774650  0.803627  0.221665
2  0.584637  0.147264  1.057825
3 -0.284136  0.912395  1.552808
4  0.189376 -0.109830  0.539341
          a         b         c
0  0.592591 -0.155407 -1.356475
1  0.833837  1.524249  1.606722
2 -0.029487 -0.051359  1.700152
3  0.921484 -0.926347  0.979818
4  0.182380 -0.186376  0.049820

In [640]: import sqlalchemy as sa

In [641]: pd.read_sql(
   .....:     sa.text("SELECT * FROM data where Col_1=:col1"), engine, params={"col1": "X"}
   .....: )
   .....: 
Out[641]: 
   index  id                        Date Col_1  Col_2  Col_3
0      0  26  2010-10-18 00:00:00.000000     X   27.5      1

In [642]: metadata = sa.MetaData()

In [643]: data_table = sa.Table(
   .....:     "data",
   .....:     metadata,
   .....:     sa.Column("index", sa.Integer),
   .....:     sa.Column("Date", sa.DateTime),
   .....:     sa.Column("Col_1", sa.String),
   .....:     sa.Column("Col_2", sa.Float),
   .....:     sa.Column("Col_3", sa.Boolean),
   .....: )
   .....: 

In [644]: pd.read_sql(sa.select([data_table]).where(data_table.c.Col_3 is True), engine)
Out[644]: 
Empty DataFrame
Columns: [index, Date, Col_1, Col_2, Col_3]
Index: []

In [645]: import datetime as dt

In [646]: expr = sa.select([data_table]).where(data_table.c.Date > sa.bindparam("date"))

In [647]: pd.read_sql(expr, engine, params={"date": dt.datetime(2010, 10, 18)})
Out[647]: 
   index       Date Col_1  Col_2  Col_3
0      1 2010-10-19     Y -12.50  False
1      2 2010-10-20     Z   5.73   True

In [648]: df = pd.DataFrame(np.random.randn(10, 2), columns=list("AB"))

In [649]: df.to_stata("stata.dta")

In [650]: pd.read_stata("stata.dta")
Out[650]: 
   index         A         B
0      0 -1.690072  0.405144
1      1 -1.511309 -1.531396
2      2  0.572698 -1.106845
3      3 -1.185859  0.174564
4      4  0.603797 -1.796129
5      5 -0.791679  1.173795
6      6 -0.277710  1.859988
7      7 -0.258413  1.251808
8      8  1.443262  0.441553
9      9  1.168163 -2.054946

In [651]: with pd.read_stata("stata.dta", chunksize=3) as reader:
   .....:     for df in reader:
   .....:         print(df.shape)
   .....: 
(3, 3)
(3, 3)
(3, 3)
(1, 3)

In [652]: with pd.read_stata("stata.dta", iterator=True) as reader:
   .....:     chunk1 = reader.read(5)
   .....:     chunk2 = reader.read(5)
   .....: