Pandas dtypes

import pandas as pd
import numpy as np

# ===== Default inference =====
df = pd.DataFrame({
    'id': [1, 2, 3],
    'name': ['Alex', 'Sam', None],
    'score': [9.5, 8.0, None],
    'active': [True, False, True],
    'date': ['2024-04-10', '2024-04-11', '2024-04-12'],
})
print(df.dtypes)
# id        int64
# name      object
# score     float64
# active    bool
# date      object        <- string by default!

# Parse dates explicitly:
df['date'] = pd.to_datetime(df['date'])
# date    datetime64[ns]

# ===== Common dtypes =====
# int8/16/32/64           signed integers
# uint8/16/32/64          unsigned integers
# float32/64               floats
# bool                     boolean
# object                   Python objects (usually strings)
# string[python]           dedicated string dtype (Pandas 1.0+)
# category                 enum-like; saves memory + speeds up groupby
# datetime64[ns]           datetimes (tz-naive)
# datetime64[ns, tz]       tz-aware
# timedelta64[ns]          durations

# Nullable types (Pandas 1.0+; capital first letter):
# Int8, Int16, Int32, Int64        nullable ints
# UInt8, ..., UInt64
# Float32, Float64
# boolean                            nullable bool
# string                             nullable string

# ===== Why nullable types matter =====
# Standard int64 cannot hold NaN. If you have nulls, Pandas upcasts to float64.
# Use Int64 (capital) to keep integer semantics with NaN support:
df['id_nullable'] = pd.array([1, None, 3], dtype='Int64')

# ===== Cast =====
df['score'] = df['score'].astype('float32')
df['name'] = df['name'].astype('string')

# Convert dates:
df['date'] = pd.to_datetime(df['date'])

# Convert to category (saves a LOT of memory on repeated strings):
df['city'] = df['city'].astype('category')

# Numeric coercion (errors -> NaN):
df['x'] = pd.to_numeric(df['x'], errors='coerce')

# ===== Inspect memory =====
df.memory_usage(deep=True)
df.info(memory_usage='deep')

# ===== Category dtype =====
cats = pd.Series(['a', 'b', 'a', 'c'], dtype='category')
cats.cat.categories          # Index(['a', 'b', 'c'])
cats.cat.codes               # 0, 1, 0, 2 (compact)

# Use ordered categories for proper sorting:
sizes = pd.Categorical(['S', 'L', 'M'], categories=['S', 'M', 'L'], ordered=True)
pd.Series(sizes).sort_values()

# ===== Reading with explicit dtypes =====
# Faster + less memory if you know the schema:
df = pd.read_csv('data.csv',
                 dtype={'id': 'Int64', 'name': 'string', 'score': 'float32', 'city': 'category'},
                 parse_dates=['date'])

# ===== Arrow-backed dtypes (Pandas 2+) =====
# pip install pyarrow
df = pd.read_csv('data.csv', dtype_backend='pyarrow')
# string[pyarrow], int64[pyarrow], etc.
# Faster + smaller than NumPy-backed object strings.

# ===== Datetime details =====
df['date'].dt.year
df['date'].dt.day_name()
df['date'].dt.tz_localize('UTC')          # add tz
df['date'].dt.tz_convert('Australia/Sydney')

# ===== Patterns to internalise =====
# - Explicit dtypes on read_csv -> faster + smaller
# - 'category' for low-cardinality strings
# - 'Int64' (nullable) over int64 + NaN -> float surprise
# - 'datetime64[ns, UTC]' for tz-aware times

# ===== Pitfalls =====
# - 'object' dtype for strings -> slow + memory-hungry; use 'string' or pyarrow
# - Implicit upcasting (int + NaN -> float) silently broadens
# - Category + new values added without registering them -> KeyError
# - dt accessor on object dtype -> errors; convert with pd.to_datetime first

import pandas as pd
df['age']  = df['age'].astype('Int64')   # nullable int
df['date'] = pd.to_datetime(df['date'])
df['cat']  = df['cat'].astype('category')