Pandas
{Back to Index} Expand All Details on Page

data = {'Ohio': 35000, 'Texas': 71000, 'Oregon': 16000, 'Utah': 5000}
states = ['California', 'Ohio', 'Oregon', 'Texas']
s = pd.Series(data, index=states)
s.name = 'population'
s.index.name = 'state'
log("s", s)

====================================== s =======================================
state
California        NaN
Ohio          35000.0
Oregon        16000.0
Texas         71000.0
Name: population, dtype: float64

s = pd.Series(np.random.randn(5), index=['a', 'b', 'c', 'd', 'e'])
log("s", s)
log("s.values", s.values)
log("s.index", s.index)

====================================== s =======================================
a    0.007460
b   -0.346364
c   -1.524387
d   -0.389066
e    0.464790
dtype: float64
=================================== s.values ===================================
[ 0.00745967 -0.34636371 -1.52438655 -0.38906608  0.46479046]
=================================== s.index ====================================
Index(['a', 'b', 'c', 'd', 'e'], dtype='object')

相同索引值才进行操作

s1 = pd.Series(np.random.randint(3, size=3), index=['a', 'c', 'e'])
s2 = pd.Series(np.random.randint(3, size=3), index=['a', 'd', 'e'])
log("s1", s1)
log("s2", s2)
log("s1 + s2", s1 + s2)

====================================== s1 ======================================
a    1
c    0
e    1
dtype: int64
====================================== s2 ======================================
a    1
d    1
e    0
dtype: int64
=================================== s1 + s2 ====================================
a    2.0
c    NaN
d    NaN
e    1.0
dtype: float64

s = pd.Series([4.5, 7.2, -5.3, 3.6], index=['d', 'b', 'a', 'c'])
log("s", s)
log("s.reindex(['a', 'b', 'c', 'd', 'e'])", s.reindex(['a', 'b', 'c', 'd', 'e']))

====================================== s =======================================
d    4.5
b    7.2
a   -5.3
c    3.6
dtype: float64
===================== s.reindex(['a', 'b', 'c', 'd', 'e']) =====================
a   -5.3
b    7.2
c    3.6
d    4.5
e    NaN
dtype: float64

重建索引时插值:

s = pd.Series(['blue', 'purple', 'yellow'], index=[0, 2, 4])
log("s", s)
log("s.reindex(range(6), method='ffill')", s.reindex(range(6), method='ffill'))

====================================== s =======================================
0      blue
2    purple
4    yellow
dtype: object
===================== s.reindex(range(6), method='ffill') ======================
0      blue
1      blue
2    purple
3    purple
4    yellow
5    yellow
dtype: object

s = pd.Series(np.arange(5.), index=list('abcde'))
log("s", s)
log("s.drop('c')", s.drop('c'))
log("s.drop(['d', 'c'])", s.drop(['d', 'c']))

====================================== s =======================================
a    0.0
b    1.0
c    2.0
d    3.0
e    4.0
dtype: float64
================================= s.drop('c') ==================================
a    0.0
b    1.0
d    3.0
e    4.0
dtype: float64
============================== s.drop(['d', 'c']) ==============================
a    0.0
b    1.0
e    4.0
dtype: float64

s = pd.Series([4, 7, -5, 3], index=['a', 'b', 'c', 'd'])
log("s", s)
log("s['a']", s['a'])
s['d'] = 6
log("s[['c', 'a', 'd']]", s[['c', 'a', 'd']])
log("s['b' : 'c']", s['b' : 'c'])

====================================== s =======================================
a    4
b    7
c   -5
d    3
dtype: int64
==================================== s['a'] ====================================
4
============================== s[['c', 'a', 'd']] ==============================
c   -5
a    4
d    6
dtype: int64
================================= s['b' : 'c'] =================================
b    7
c   -5
dtype: int64

s = pd.Series([4, 7, -5, 3], index=['d', 'b', 'a', 'c'])
log("s[s > 0]", s[s > 0])

=================================== s[s > 0] ===================================
d    4
b    7
c    3
dtype: int64

data = {'Nevada': {2001: 2.4, 2002: 2.9},
	'Ohio': {2001: 1.5, 2001: 1.7, 2002: 3.6}}
df = pd.DataFrame(data)
df.index.name = 'year'
df.columns.name = 'state'
log("df", df)

====================================== df ======================================
state  Nevada  Ohio
year
2001      2.4   1.7
2002      2.9   3.6

在 DataFrame 中， reindex 可以改变行索引，列索引，也可以同时改变两者。

fill method 只对行重新索引有效，不适用列。

df = pd.DataFrame(np.arange(9).reshape((3,3)),
		  index=list('acd'),
		  columns=['Ohio', 'Texas', 'California'])
log("df", df)
# 重建行索引
log("df.reindex(['a', 'b', 'c', 'd'])", df.reindex(['a', 'b', 'c', 'd']))
# 重建列索引
states = ['Texas', 'Utah', 'California']
log("df.reindex(columns=states)", df.reindex(columns=states))

====================================== df ======================================
   Ohio  Texas  California
a     0      1           2
c     3      4           5
d     6      7           8
======================= df.reindex(['a', 'b', 'c', 'd']) =======================
   Ohio  Texas  California
a   0.0    1.0         2.0
b   NaN    NaN         NaN
c   3.0    4.0         5.0
d   6.0    7.0         8.0
========================== df.reindex(columns=states) ==========================
   Texas  Utah  California
a      1   NaN           2
c      4   NaN           5
d      7   NaN           8

DataFrame 在进行数据计算时， 会自动按行和列进行数据对齐 。 最终的计算结果会合并两个 DataFrame 。

df1 = pd.DataFrame(np.arange(9.).reshape((3, 3)),
		   columns=list('bcd'),
		   index=['Ohio', 'Texas', 'Colorado'])

df2 = pd.DataFrame(np.arange(12.).reshape((4, 3)),
		   columns=list('bde'),
		   index=['Utah', 'Ohio', 'Texas', 'Oregon'])

log("df1", df1)
log("df2", df2)
log("df1 + df2", df1 + df2)

当将列表或数组赋值给一个列时，长度必须和 DataFrame 的长度相匹配。

data = {'state': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada', 'Nevada'],
	'year': [2000, 2001, 2002, 2001, 2002, 2003],
	'pop': [1.5, 1.7, 3.6, 2.4, 2.9, 3.2]}
columns = ['year', 'state', 'pop']
index = ['one', 'two', 'three', 'four', 'five', 'six']
df = pd.DataFrame(data, columns=columns, index=index)
df['debt'] = 16.5 # 标量赋值
df['income'] = np.arange(6.) # 数组赋值
log("df", df)

====================================== df ======================================
       year   state  pop  debt  income
one    2000    Ohio  1.5  16.5     0.0
two    2001    Ohio  1.7  16.5     1.0
three  2002    Ohio  3.6  16.5     2.0
four   2001  Nevada  2.4  16.5     3.0
five   2002  Nevada  2.9  16.5     4.0
six    2003  Nevada  3.2  16.5     5.0

将 Series 赋值给一列时，Series 的索引会按照 DataFrame 的索引重新排列。

val = pd.Series([-1.2, -1.5, -1.7], index=['two', 'four', 'five'])
df['debt'] = val
log("df", df)

====================================== df ======================================
       year   state  pop  debt  income
one    2000    Ohio  1.5   NaN     0.0
two    2001    Ohio  1.7  -1.2     1.0
three  2002    Ohio  3.6   NaN     2.0
four   2001  Nevada  2.4  -1.5     3.0
five   2002  Nevada  2.9  -1.7     4.0
six    2003  Nevada  3.2   NaN     5.0

data = {'state': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada', 'Nevada'],
	'year': [2000, 2001, 2002, 2001, 2002, 2003],
	'pop': [1.5, 1.7, 3.6, 2.4, 2.9, 3.2]}
columns = ['year', 'state', 'pop']
index = ['one', 'two', 'three', 'four', 'five', 'six']
df = pd.DataFrame(data, columns=columns, index=index)
del df['pop']
yearSeries = df.pop('year')
df.drop('state', axis='columns', inplace=True)
log("df", df)

====================================== df ======================================
Empty DataFrame
Columns: []
Index: [one, two, three, four, five, six]

df = pd.DataFrame(np.arange(16).reshape((4, 4)),
		  index=["Ohio", "Colorado", "Utah", "New York"],
		  columns=["one", "two", "three", "four"])
log("df", df)
log("df.drop('Colorado')", df.drop('Colorado')) # 删除行
log("df.drop('two', axis='columns')", df.drop('two', axis='columns'))

====================================== df ======================================
          one  two  three  four
Ohio        0    1      2     3
Colorado    4    5      6     7
Utah        8    9     10    11
New York   12   13     14    15
============================= df.drop('Colorado') ==============================
          one  two  three  four
Ohio        0    1      2     3
Utah        8    9     10    11
New York   12   13     14    15
======================== df.drop('two', axis='columns') ========================
          one  three  four
Ohio        0      2     3
Colorado    4      6     7
Utah        8     10    11
New York   12     14    15

从 DataFrame 中玄虚的列是数据的 视图 ，而不是拷贝。 如果需要复制，应当显式地使用 Series 的 copy 方法。

返回的 Series 与原 DataFrame 有相同的索引，且 Series 的 name 属性也会被合理地设置。

data = {'state': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada', 'Nevada'],
	'year': [2000, 2001, 2002, 2001, 2002, 2003],
	'pop': [1.5, 1.7, 3.6, 2.4, 2.9, 3.2]}
columns = ['year', 'state', 'pop']
df = pd.DataFrame(data, columns=columns)
log("df", df)
log("df['state']", df['state'])
log("df.year", df.year)
log("df[['state', 'pop']]", df[['state', 'pop']])

====================================== df ======================================
   year   state  pop
0  2000    Ohio  1.5
1  2001    Ohio  1.7
2  2002    Ohio  3.6
3  2001  Nevada  2.4
4  2002  Nevada  2.9
5  2003  Nevada  3.2
================================= df['state'] ==================================
0      Ohio
1      Ohio
2      Ohio
3    Nevada
4    Nevada
5    Nevada
Name: state, dtype: object
=================================== df.year ====================================
0    2000
1    2001
2    2002
3    2001
4    2002
5    2003
Name: year, dtype: int64
============================= df[['state', 'pop']] =============================
    state  pop
0    Ohio  1.5
1    Ohio  1.7
2    Ohio  3.6
3  Nevada  2.4
4  Nevada  2.9
5  Nevada  3.2

df = pd.DataFrame(np.random.randn(6, 4),
		  index=list('ABCDEF'),
		  columns=['one', 'two', 'three', 'four'])
log("df", df)
log("df.filter(regex=r'^t.*$')", df.filter(regex=r'^t.*$'))

====================================== df ======================================
        one       two     three      four
A  0.266558  0.390929  0.381822 -0.662022
B  0.947612  1.492351  1.824414 -0.682042
C  0.920167 -0.387809 -1.606654 -0.692762
D -0.491672  0.135303  1.653127  0.036277
E -0.922068  0.128126 -1.823203  0.054199
F -0.023060 -0.725380  0.062327 -0.608580
========================== df.filter(regex=r'^t.*$') ===========================
        two     three
A  0.390929  0.381822
B  1.492351  1.824414
C -0.387809 -1.606654
D  0.135303  1.653127
E  0.128126 -1.823203
F -0.725380  0.062327

df = pd.DataFrame(np.arange(16).reshape((4, 4)),
		  index=["Ohio", "Colorado", "Utah", "New York"],
		  columns=["one", "two", "three", "four"])
log("df", df)
log("df[:2]", df[:2])

====================================== df ======================================
          one  two  three  four
Ohio        0    1      2     3
Colorado    4    5      6     7
Utah        8    9     10    11
New York   12   13     14    15
==================================== df[:2] ====================================
          one  two  three  four
Ohio        0    1      2     3
Colorado    4    5      6     7

df = pd.DataFrame(np.arange(16).reshape((4, 4)),
		  index=["Ohio", "Colorado", "Utah", "New York"],
		  columns=["one", "two", "three", "four"])
log("df", df)
log("df[df['three'] > 5]", df[df['three'] > 5])

====================================== df ======================================
          one  two  three  four
Ohio        0    1      2     3
Colorado    4    5      6     7
Utah        8    9     10    11
New York   12   13     14    15
============================= df[df['three'] > 5] ==============================
          one  two  three  four
Colorado    4    5      6     7
Utah        8    9     10    11
New York   12   13     14    15

df = pd.DataFrame(np.arange(16).reshape((4, 4)),
		  index=["Ohio", "Colorado", "Utah", "New York"],
		  columns=["one", "two", "three", "four"])
log("df", df)
log("df.loc['Colorado', ['two', 'three']]", df.loc['Colorado', ['two', 'three']])
log("df.loc[:'Utah', 'two']", df.loc[:'Utah', 'two'])

====================================== df ======================================
          one  two  three  four
Ohio        0    1      2     3
Colorado    4    5      6     7
Utah        8    9     10    11
New York   12   13     14    15
===================== df.loc['Colorado', ['two', 'three']] =====================
two      5
three    6
Name: Colorado, dtype: int64
============================ df.loc[:'Utah', 'two'] ============================
Ohio        1
Colorado    5
Utah        9
Name: two, dtype: int64

df = pd.DataFrame(np.arange(16).reshape((4, 4)),
		  index=["Ohio", "Colorado", "Utah", "New York"],
		  columns=["one", "two", "three", "four"])
log("df", df)
log("df.iloc[2, [3, 0, 1]]", df.iloc[2, [3, 0, 1]])
log("df.iloc[[1, 2], [3, 0, 1]]", df.iloc[[1, 2], [3, 0, 1]])
log("df.iloc[:, :3][df.three > 5]", df.iloc[:, :3][df.three > 5])

====================================== df ======================================
          one  two  three  four
Ohio        0    1      2     3
Colorado    4    5      6     7
Utah        8    9     10    11
New York   12   13     14    15
============================ df.iloc[2, [3, 0, 1]] =============================
four    11
one      8
two      9
Name: Utah, dtype: int64
========================== df.iloc[[1, 2], [3, 0, 1]] ==========================
          four  one  two
Colorado     7    4    5
Utah        11    8    9
========================= df.iloc[:, :3][df.three > 5] =========================
          one  two  three
Colorado    4    5      6
Utah        8    9     10
New York   12   13     14

df = pd.DataFrame(np.arange(16).reshape((4, 4)),
		  index=["Ohio", "Colorado", "Utah", "New York"],
		  columns=["one", "two", "three", "four"])
log("df", df)
log("df.sample(frac=0.5)", df.sample(frac=0.5))  # Randomly
log("df.sample(n=3)", df.sample(n=3))  # Randomly

====================================== df ======================================
          one  two  three  four
Ohio        0    1      2     3
Colorado    4    5      6     7
Utah        8    9     10    11
New York   12   13     14    15
============================= df.sample(frac=0.5) ==============================
          one  two  three  four
Ohio        0    1      2     3
New York   12   13     14    15
================================ df.sample(n=3) ================================
          one  two  three  four
Colorado    4    5      6     7
Ohio        0    1      2     3
Utah        8    9     10    11

df = pd.DataFrame(np.random.randn(6, 4),
		  index=list('ABCDEF'),
		  columns=['one', 'two', 'three', 'four'])
log("origin df", df)
df.rename(columns={'three': 'san'}, inplace=True)
log("df (column index renamed)", df)
df.rename(index={'C': 'ccc'}, inplace=True)
log("df (index renamed)", df)

================================== origin df ===================================
        one       two     three      four
A -1.293186 -1.303202 -0.434815  1.157079
B  0.559491  1.651170  1.130642  0.372430
C  0.087382  0.948737  0.103419 -0.364204
D  0.363415  0.463077 -1.130338 -1.252423
E -0.570689 -1.141226  0.144087 -0.297187
F  1.028409  1.627355 -1.264463 -0.109870
========================== df (column index renamed) ===========================
        one       two       san      four
A -1.293186 -1.303202 -0.434815  1.157079
B  0.559491  1.651170  1.130642  0.372430
C  0.087382  0.948737  0.103419 -0.364204
D  0.363415  0.463077 -1.130338 -1.252423
E -0.570689 -1.141226  0.144087 -0.297187
F  1.028409  1.627355 -1.264463 -0.109870
============================== df (index renamed) ==============================
          one       two       san      four
A   -1.293186 -1.303202 -0.434815  1.157079
B    0.559491  1.651170  1.130642  0.372430
ccc  0.087382  0.948737  0.103419 -0.364204
D    0.363415  0.463077 -1.130338 -1.252423
E   -0.570689 -1.141226  0.144087 -0.297187
F    1.028409  1.627355 -1.264463 -0.109870

s = pd.Series([1, np.nan, 3.5, np.nan, 7])
log("s.dropna()", s.dropna()) # 等价于 s[s.notnull()]

================================== s.dropna() ==================================
0    1.0
2    3.5
4    7.0
dtype: float64

当处理 DataFrame 对象时， dropna 默认情况下删除包含缺失值的行 ： （使用 axis = 1 删除列）

df = pd.DataFrame([
    [1, 6.5, 3],
    [1, np.nan, np.nan],
    [np.nan, np.nan, np.nan],
    [np.nan, 6.5, 3]
])
log("df", df)
log("df.dropna()", df.dropna())

====================================== df ======================================
     0    1    2
0  1.0  6.5  3.0
1  1.0  NaN  NaN
2  NaN  NaN  NaN
3  NaN  6.5  3.0
================================= df.dropna() ==================================
     0    1    2
0  1.0  6.5  3.0

当传入 how = 'all' 时， 删除所有值均为 NA 的那些行 ：

log("df.dropna(how='all')", df.dropna(how='all'))

使用 thresh 参数来保留 包含一定数量观察值 的行：

df = pd.DataFrame(np.random.randn(7, 3))
df.iloc[:4, 1] = np.nan
df.iloc[:2, 2] = np.nan
log("df", df)
log("df.dropna()", df.dropna())
log("df.dropna(thresh=2)", df.dropna(thresh=2))

====================================== df ======================================
          0         1         2
0 -0.215937       NaN       NaN
1 -1.358234       NaN       NaN
2  0.331335       NaN -0.901148
3 -0.466495       NaN  2.392151
4 -0.178190  0.446226  0.710607
5 -0.446093 -0.317979 -0.601814
6  0.550952  1.036072 -1.812287
================================= df.dropna() ==================================
          0         1         2
4 -0.178190  0.446226  0.710607
5 -0.446093 -0.317979 -0.601814
6  0.550952  1.036072 -1.812287
============================= df.dropna(thresh=2) ==============================
          0         1         2
2  0.331335       NaN -0.901148
3 -0.466495       NaN  2.392151
4 -0.178190  0.446226  0.710607
5 -0.446093 -0.317979 -0.601814
6  0.550952  1.036072 -1.812287

df = pd.DataFrame(np.random.randn(7, 3))
df.iloc[:4, 1] = np.nan
df.iloc[:2, 2] = np.nan
log("df", df)
log("df.fillna(0)", df.fillna(0))

====================================== df ======================================
          0         1         2
0  0.997841       NaN       NaN
1 -0.554321       NaN       NaN
2 -0.804640       NaN  0.838684
3  0.666262       NaN -1.009344
4  0.049296 -0.091335 -0.724490
5  0.172516 -0.255515  0.760672
6  3.210886  1.221200  0.913991
================================= df.fillna(0) =================================
          0         1         2
0  0.997841  0.000000  0.000000
1 -0.554321  0.000000  0.000000
2 -0.804640  0.000000  0.838684
3  0.666262  0.000000 -1.009344
4  0.049296 -0.091335 -0.724490
5  0.172516 -0.255515  0.760672
6  3.210886  1.221200  0.913991

调用 fillna 时使用字典，可以 为不同列设定不同的填充值 ：

log("df.fillna({1: 0.5, 2: 0})", df.fillna({1: 0.5, 2: 0}))

========================== df.fillna({1: 0.5, 2: 0}) ===========================
          0         1         2
0  0.997841  0.500000  0.000000
1 -0.554321  0.500000  0.000000
2 -0.804640  0.500000  0.838684
3  0.666262  0.500000 -1.009344
4  0.049296 -0.091335 -0.724490
5  0.172516 -0.255515  0.760672
6  3.210886  1.221200  0.913991

使用 插值方法 ：

df = pd.DataFrame(np.random.randn(6, 3))
df.iloc[2:, 1] = np.nan
df.iloc[4:, 2] = np.nan
log("df", df)
log("df.fillna(method='ffill')", df.fillna(method='ffill'))
log("df.fillna(method='ffill', limit=2)", df.fillna(method='ffill', limit=2))

====================================== df ======================================
          0         1         2
0  0.663103  0.176675 -1.293323
1 -0.108963  0.912126  0.806356
2  0.215125       NaN -1.686062
3 -0.411193       NaN -0.738710
4 -0.038864       NaN       NaN
5 -0.561398       NaN       NaN
========================== df.fillna(method='ffill') ===========================
          0         1         2
0  0.663103  0.176675 -1.293323
1 -0.108963  0.912126  0.806356
2  0.215125  0.912126 -1.686062
3 -0.411193  0.912126 -0.738710
4 -0.038864  0.912126 -0.738710
5 -0.561398  0.912126 -0.738710
====================== df.fillna(method='ffill', limit=2) ======================
          0         1         2
0  0.663103  0.176675 -1.293323
1 -0.108963  0.912126  0.806356
2  0.215125  0.912126 -1.686062
3 -0.411193  0.912126 -0.738710
4 -0.038864       NaN -0.738710
5 -0.561398       NaN -0.738710

df = pd.DataFrame(
    {
	"k1": ['one', 'two'] * 3 + ['two'],
	"k2": [1, 1, 2, 3, 3, 4, 4]
    }
)
log("df", df)
log("df.duplicated()", df.duplicated())

====================================== df ======================================
    k1  k2
0  one   1
1  two   1
2  one   2
3  two   3
4  one   3
5  two   4
6  two   4
=============================== df.duplicated() ================================
0    False
1    False
2    False
3    False
4    False
5    False
6     True
dtype: bool

drop_duplicates 返回的是 DataFrame ，内容是 duplicated 返回 Series 中为 False 的部分：

log("df.drop_duplicates()", df.drop_duplicates())

============================= df.drop_duplicates() =============================
    k1  k2
0  one   1
1  two   1
2  one   2
3  two   3
4  one   3
5  two   4

基于某列 去除重复值：

df['v1'] = range(7)
log("df", df)
log("df.drop_duplicates(['k1'])", df.drop_duplicates(['k1']))

====================================== df ======================================
    k1  k2  v1
0  one   1   0
1  two   1   1
2  one   2   2
3  two   3   3
4  one   3   4
5  two   4   5
6  two   4   6
========================== df.drop_duplicates(['k1']) ==========================
    k1  k2  v1
0  one   1   0
1  two   1   1

drop_duplicates 默认保留第一个观测值，传入参数 keep = 'last' 将返回最后一个：

log("df.drop_duplicates(['k1', 'k2'], keep='last')", df.drop_duplicates(['k1', 'k2'], keep='last'))

================ df.drop_duplicates(['k1', 'k2'], keep='last') =================
    k1  k2  v1
0  one   1   0
1  two   1   1
2  one   2   2
3  two   3   3
4  one   3   4
6  two   4   6

  df = pd.DataFrame(
      {
	  "food": ['bacon',
		   'pulled pork',
		   'bacon',
		   'Pastrami',
		   'corned beef',
		   'Bacon',
		   'pastrami',
		   'honey ham',
		   'nova lox'],
	  "ounces": [4, 3, 12, 6, 7.5, 8, 3, 5, 6]
      }
  )

  meat_to_animal = {
      "bacon": 'pig',
      "pulled pork": 'pig',
      "pastrami": 'cow',
      "corned beef": 'cow',
      "honey ham": 'pig',
      "nova lox": 'salmon'
  }

log("df", df)
# 将值转换为小写
lowercased = df['food'].str.lower()
log("lowercased", lowercased)
df['animal'] = lowercased.map(meat_to_animal)
log("df", df)
# 也可以 data['food'].map(lambda x: meat_to_animal[x.lower()])

====================================== df ======================================
          food  ounces
0        bacon     4.0
1  pulled pork     3.0
2        bacon    12.0
3     Pastrami     6.0
4  corned beef     7.5
5        Bacon     8.0
6     pastrami     3.0
7    honey ham     5.0
8     nova lox     6.0
================================== lowercased ==================================
0          bacon
1    pulled pork
2          bacon
3       pastrami
4    corned beef
5          bacon
6       pastrami
7      honey ham
8       nova lox
Name: food, dtype: object
====================================== df ======================================
          food  ounces  animal
0        bacon     4.0     pig
1  pulled pork     3.0     pig
2        bacon    12.0     pig
3     Pastrami     6.0     cow
4  corned beef     7.5     cow
5        Bacon     8.0     pig
6     pastrami     3.0     cow
7    honey ham     5.0     pig
8     nova lox     6.0  salmon

使用 fillna() 是通用值替换的特殊案例。

使用 map() 可以用来修改一个数据中的值，但是 replace() 提供了 更为简单 的实现。

s = pd.Series([1, -999, 2, -999, -1000, 3])
log("s", s)
log("s.replace(-999, np.nan)", s.replace(-999, np.nan))

====================================== s =======================================
0       1
1    -999
2       2
3    -999
4   -1000
5       3
dtype: int64
=========================== s.replace(-999, np.nan) ============================
0       1.0
1       NaN
2       2.0
3       NaN
4   -1000.0
5       3.0
dtype: float64

如果想要 一次替代多个值 ，可以传入一个列表：

log("s.replace([-999, -1000], np.nan)", s.replace([-999, -1000], np.nan))

======================= s.replace([-999, -1000], np.nan) =======================
0    1.0
1    NaN
2    2.0
3    NaN
4    NaN
5    3.0
dtype: float64

将不同的值替换为不同的值 ：

log("s.replace([-999, -1000], [np.nan, 0])", s.replace([-999, -1000], [np.nan, 0]))
log("s.replace({-999: np.nan, -1000: 0})", s.replace({-999: np.nan, -1000: 0}))

==================== s.replace([-999, -1000], [np.nan, 0]) =====================
0    1.0
1    NaN
2    2.0
3    NaN
4    0.0
5    3.0
dtype: float64
===================== s.replace({-999: np.nan, -1000: 0}) ======================
0    1.0
1    NaN
2    2.0
3    NaN
4    0.0
5    3.0
dtype: float64

df = pd.DataFrame(np.arange(12).reshape((3,4)),
		  index=['Ohio', 'Colorado', 'New York'],
		  columns=['one', 'two', 'three', 'four'])

df.index = df.index.map(lambda x: x[:4].upper())
log("df", df)
log("df.rename(index=str.title, columns=str.upper)", df.rename(index=str.title, columns=str.upper))
log("df.rename(index={'OHIO': 'INDIANA'}, columns={'three': 'peekaboo'})", df.rename(index={'OHIO': 'INDIANA'}, columns={'three': 'peekaboo'}))

====================================== df ======================================
      one  two  three  four
OHIO    0    1      2     3
COLO    4    5      6     7
NEW     8    9     10    11
================ df.rename(index=str.title, columns=str.upper) =================
      ONE  TWO  THREE  FOUR
Ohio    0    1      2     3
Colo    4    5      6     7
New     8    9     10    11
===== df.rename(index={'OHIO': 'INDIANA'}, columns={'three': 'peekaboo'}) ======
         one  two  peekaboo  four
INDIANA    0    1         2     3
COLO       4    5         6     7
NEW        8    9        10    11

连续值经常需要离散化，或者分离成 箱子 进行分析。

ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]
bins = [18, 25, 35, 60, 100]
cats = pd.cut(ages, bins)
log("cats", cats)
log("cats.codes", cats.codes)
log("cats.categories", cats.categories)
log("cats.value_counts()", cats.value_counts())

===================================== cats =====================================
[(18, 25], (18, 25], (18, 25], (25, 35], (18, 25], ..., (25, 35], (60, 100], (35, 60], (35, 60], (25, 35]]
Length: 12
Categories (4, interval[int64]): [(18, 25] < (25, 35] < (35, 60] < (60, 100]]
================================== cats.codes ==================================
[0 0 0 1 0 0 2 1 3 2 2 1]
=============================== cats.categories ================================
IntervalIndex([(18, 25], (25, 35], (35, 60], (60, 100]]
              closed='right',
              dtype='interval[int64]')
============================= cats.value_counts() ==============================
(18, 25]     5
(25, 35]     3
(35, 60]     3
(60, 100]    1
dtype: int64

通过传递 right = False 来改变区间哪一边是封闭的。

自定义箱名 ：

group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']
log("pd.cut(ages, bins, labels=group_names)", pd.cut(ages, bins, labels=group_names))

==================== pd.cut(ages, bins, labels=group_names) ====================
[Youth, Youth, Youth, YoungAdult, Youth, ..., YoungAdult, Senior, MiddleAged, MiddleAged, YoungAdult]
Length: 12
Categories (4, object): [Youth < YoungAdult < MiddleAged < Senior]

若 使用分箱的个数代替箱边 ， pandas 将根据数据中的最小值和最大值计算出等长的箱：

data = np.random.rand(20)
# precision 表示将十进制精度限制在两位
log("pd.cut(data, 4, precision=2)", pd.cut(data, 4, precision=2))

========================= pd.cut(data, 4, precision=2) =========================
[(0.5, 0.73], (0.5, 0.73], (0.73, 0.97], (0.26, 0.5], (0.024, 0.26], ..., (0.024, 0.26], (0.26, 0.5], (0.73, 0.97], (0.5, 0.73], (0.73, 0.97]]
Length: 20
Categories (4, interval[float64]): [(0.024, 0.26] < (0.26, 0.5] < (0.5, 0.73] < (0.73, 0.97]]

使用 cut 通常不会使每个箱具有相同的数据量。 qcut 基于样本分位数进行分箱，可以获得 等长的箱 ：

data = np.random.randn(1000)
qcats = pd.qcut(data, 4)
log("qcats.value_counts()", qcats.value_counts())

============================= qcats.value_counts() =============================
(-3.532, -0.671]    250
(-0.671, 0.0582]    250
(0.0582, 0.744]     250
(0.744, 3.077]      250
dtype: int64

自定义分位数 ：(0 和 1 之间的数字)

qcuts = pd.qcut(data, [0, 0.1, 0.5, 0.9, 1])
log("qcuts.value_counts()", qcuts.value_counts())

============================= qcuts.value_counts() =============================
(-3.532, -1.302]    100
(-1.302, 0.0582]    400
(0.0582, 1.374]     400
(1.374, 3.077]      100
dtype: int64

df = pd.DataFrame(np.random.randn(5, 4))
log("df", df)
log("np.abs(df) > 1", np.abs(df) > 1)
log("(np.abs(df) > 1).any()", (np.abs(df) > 1).any())
log("(np.abs(df) > 1).any(axis=1)", (np.abs(df) > 1).any(axis=1))
log("(np.abs(df) > 1).any().any()", (np.abs(df) > 1).any().any())

====================================== df ======================================
          0         1         2         3
0  0.634347 -0.504562  0.337217  0.191453
1 -0.378714 -1.212778 -0.194362 -2.113623
2 -1.422029  0.532520 -0.348306  0.059880
3 -1.184722 -0.297884 -0.620028  0.378589
4  0.846286 -0.714991 -0.127334 -0.089754
================================ np.abs(df) > 1 ================================
       0      1      2      3
0  False  False  False  False
1  False   True  False   True
2   True  False  False  False
3   True  False  False  False
4  False  False  False  False
============================ (np.abs(df) > 1).any() ============================
0     True
1     True
2    False
3     True
dtype: bool
========================= (np.abs(df) > 1).any(axis=1) =========================
0    False
1     True
2     True
3     True
4    False
dtype: bool
========================= (np.abs(df) > 1).any().any() =========================
True

df = pd.DataFrame(np.arange(5*4).reshape((5, 4)))
sampler = np.random.permutation(5)
log("df", df)
log("sampler", sampler)
log("df.take(sampler)", df.take(sampler)) # 和 iloc 类似

log("df.sample(3)", df.sample(3))
log("df.sample(3, replace=True)", df.sample(3, replace=True)) # 允许有重复

====================================== df ======================================
    0   1   2   3
0   0   1   2   3
1   4   5   6   7
2   8   9  10  11
3  12  13  14  15
4  16  17  18  19
=================================== sampler ====================================
[2 3 0 1 4]
=============================== df.take(sampler) ===============================
    0   1   2   3
2   8   9  10  11
3  12  13  14  15
0   0   1   2   3
1   4   5   6   7
4  16  17  18  19
================================= df.sample(3) =================================
   0  1   2   3
2  8  9  10  11
0  0  1   2   3
1  4  5   6   7
========================== df.sample(3, replace=True) ==========================
    0   1   2   3
3  12  13  14  15
3  12  13  14  15
0   0   1   2   3

如果一列有 k 个不同的值，可以衍生一个k 列的，值为 0 或 1 的矩阵。

df = pd.DataFrame({'key': list('bbacab'), 'data': range(6)})
log("df", df)
log("pd.get_dummies(df['key'])", pd.get_dummies(df['key']))

====================================== df ======================================
  key  data
0   b     0
1   b     1
2   a     2
3   c     3
4   a     4
5   b     5
========================== pd.get_dummies(df['key']) ===========================
   a  b  c
0  0  1  0
1  0  1  0
2  1  0  0
3  0  0  1
4  1  0  0
5  0  1  0

添加前缀：

dummies = pd.get_dummies(df['key'], prefix='key')
log("dummies", dummies)
df_with_dummies = df[['data']].join(dummies)
log("df_with_dummies", df_with_dummies)

=================================== dummies ====================================
   key_a  key_b  key_c
0      0      1      0
1      0      1      0
2      1      0      0
3      0      0      1
4      1      0      0
5      0      1      0
=============================== df_with_dummies ================================
   data  key_a  key_b  key_c
0     0      0      1      0
1     1      0      1      0
2     2      1      0      0
3     3      0      0      1
4     4      1      0      0
5     5      0      1      0

get_dummies 与 cut 等离散化函数结合使用是 统计操作中一个有用的方法 ：

np.random.seed(12345)
values = np.random.rand(10)
log("values", values)
bins = [0, 0.2, 0.4, 0.6, 0.8, 1]
log("pd.get_dummies(pd.cut(values, bins))", pd.get_dummies(pd.cut(values, bins)))

==================================== values ====================================
[0.92961609 0.31637555 0.18391881 0.20456028 0.56772503 0.5955447
 0.96451452 0.6531771  0.74890664 0.65356987]
===================== pd.get_dummies(pd.cut(values, bins)) =====================
   (0.0, 0.2]  (0.2, 0.4]  (0.4, 0.6]  (0.6, 0.8]  (0.8, 1.0]
0           0           0           0           0           1
1           0           1           0           0           0
2           1           0           0           0           0
3           0           1           0           0           0
4           0           0           1           0           0
5           0           0           1           0           0
6           0           0           0           0           1
7           0           0           0           1           0
8           0           0           0           1           0
9           0           0           0           1           0

swaplevel 接收两个层级序号或层级名称，返回一个进行了层级变更的新对象， 数据是不变的 。

df = pd.DataFrame(np.arange(12).reshape((4, 3)),
		  index=[['a', 'a', 'b', 'b'], [1, 2, 1, 2]],
		  columns=[['Ohio', 'Ohio', 'Colorado'], ['Green', 'Red', 'Green']]
)
df.index.names = ['key1', 'key2']
df.columns.names = ['state', 'color']
log("df", df)
log("df.swaplevel('key1', 'key2')", df.swaplevel('key1', 'key2'))

====================================== df ======================================
state      Ohio     Colorado
color     Green Red    Green
key1 key2
a    1        0   1        2
     2        3   4        5
b    1        6   7        8
     2        9  10       11
========================= df.swaplevel('key1', 'key2') =========================
state      Ohio     Colorado
color     Green Red    Green
key2 key1
1    a        0   1        2
2    a        3   4        5
1    b        6   7        8
2    b        9  10       11

sort_index 只能在单一层级上对数据进行排序。

log("df.sort_index(level=1)", df.sort_index(level=1))
log("df.swaplevel(0, 1).sort_index(level=0)", df.swaplevel(0, 1).sort_index(level=0))

============================ df.sort_index(level=1) ============================
state      Ohio     Colorado
color     Green Red    Green
key1 key2
a    1        0   1        2
b    1        6   7        8
a    2        3   4        5
b    2        9  10       11
==================== df.swaplevel(0, 1).sort_index(level=0) ====================
state      Ohio     Colorado
color     Green Red    Green
key2 key1
1    a        0   1        2
     b        6   7        8
2    a        3   4        5
     b        9  10       11

如果索引按照字典顺序从最外层开始排序，则数据选择性能会更好

df = pd.DataFrame({
    "a": range(7),
    "b": range(7, 0, -1),
    "c": ['one', 'one', 'one', 'two', 'two', 'two', 'two'],
    "d": [0, 1, 2, 0, 1, 2, 3]
})
log("df", df)
log("df.set_index(['c', 'd'])", df.set_index(['c', 'd']))

====================================== df ======================================
   a  b    c  d
0  0  7  one  0
1  1  6  one  1
2  2  5  one  2
3  3  4  two  0
4  4  3  two  1
5  5  2  two  2
6  6  1  two  3
=========================== df.set_index(['c', 'd']) ===========================
       a  b
c   d
one 0  0  7
    1  1  6
    2  2  5
two 0  3  4
    1  4  3
    2  5  2
    3  6  1

默认情况下，用过行索引的列会从 DataFrame 中删除，也可以选择留下：

log("df.set_index(['c', 'd'], drop=False)", df.set_index(['c', 'd'], drop=False))

===================== df.set_index(['c', 'd'], drop=False) =====================
       a  b    c  d
c   d
one 0  0  7  one  0
    1  1  6  one  1
    2  2  5  one  2
two 0  3  4  two  0
    1  4  3  two  1
    2  5  2  two  2
    3  6  1  two  3

reset_index 是 set_index 的 反操作 ，分层索引的行索引层级会被移动为列索引。

df = pd.DataFrame({
    "a": range(7),
    "b": range(7, 0, -1),
    "c": ['one', 'one', 'one', 'two', 'two', 'two', 'two'],
    "d": [0, 1, 2, 0, 1, 2, 3]
})
log("df", df)
df2 = df.set_index(['c', 'd'])
log("df2", df2)
log("df2.reset_index()", df2.reset_index())

====================================== df ======================================
   a  b    c  d
0  0  7  one  0
1  1  6  one  1
2  2  5  one  2
3  3  4  two  0
4  4  3  two  1
5  5  2  two  2
6  6  1  two  3
===================================== df2 ======================================
       a  b
c   d
one 0  0  7
    1  1  6
    2  2  5
two 0  3  4
    1  4  3
    2  5  2
    3  6  1
============================== df2.reset_index() ===============================
     c  d  a  b
0  one  0  0  7
1  one  1  1  6
2  one  2  2  5
3  two  0  3  4
4  two  1  4  3
5  two  2  5  2
6  two  3  6  1

concat 方法本质其实就是按索引合并 ，默认情况下 concat 方法是沿着 axis = 0 的轴向生效的， 生成另一个 Series ；如传递 axis = 1 ， 返回的结果则是 DataFrame 。

s1 = pd.Series([0, 1], index=['a', 'b'])
s2 = pd.Series([2, 3, 4], index=['c', 'd', 'e'])
s3 = pd.Series([5, 6], index=['f', 'g'])
log("s1", s1)
log("s2", s2)
log("s3", s3)
log("pd.concat([s1, s2, s3])", pd.concat([s1, s2, s3]))
log("pd.concat([s1, s2, s3], axis=1)", pd.concat([s1, s2, s3], axis=1))

====================================== s1 ======================================
a    0
b    1
dtype: int64
====================================== s2 ======================================
c    2
d    3
e    4
dtype: int64
====================================== s3 ======================================
f    5
g    6
dtype: int64
=========================== pd.concat([s1, s2, s3]) ============================
a    0
b    1
c    2
d    3
e    4
f    5
g    6
dtype: int64
======================= pd.concat([s1, s2, s3], axis=1) ========================
     0    1    2
a  0.0  NaN  NaN
b  1.0  NaN  NaN
c  NaN  2.0  NaN
d  NaN  3.0  NaN
e  NaN  4.0  NaN
f  NaN  NaN  5.0
g  NaN  NaN  6.0

concat 默认使用 外连接 ：

s4 = pd.concat([s1, s3])
log("s4", s4)
log("pd.concat([s1, s4], axis=1)", pd.concat([s1, s4], axis=1))
log("pd.concat([s1, s4], axis=1, join='inner')", pd.concat([s1, s4], axis=1, join='inner'))

====================================== s4 ======================================
a    0
b    1
f    5
g    6
dtype: int64
========================= pd.concat([s1, s4], axis=1) ==========================
     0  1
a  0.0  0
b  1.0  1
f  NaN  5
g  NaN  6
================== pd.concat([s1, s4], axis=1, join='inner') ===================
   0  1
a  0  0
b  1  1

combine_first 既不是合并操作，也不是连接操作，它是用来填补调用对象的缺失值：

a = pd.Series([np.nan, 2.5, 0, 3.5, 4.5, np.nan],
	      index=list('fedcba')
)
b = pd.Series([0, np.nan, 2, np.nan, np.nan, 5],
	      index=list('abcdef')
)
log("a", a)
log("b", b)
log("b.combine_first(a)", b.combine_first(a))

====================================== a =======================================
f    NaN
e    2.5
d    0.0
c    3.5
b    4.5
a    NaN
dtype: float64
====================================== b =======================================
a    0.0
b    NaN
c    2.0
d    NaN
e    NaN
f    5.0
dtype: float64
============================== b.combine_first(a) ==============================
a    0.0
b    4.5
c    2.0
d    0.0
e    2.5
f    5.0
dtype: float64

在 DataFrame 中， combine_first 逐列做相同的操作：

df1 = pd.DataFrame(
    {
	"a": [1, np.nan, 5, np.nan],
	"b": [np.nan, 2, np.nan, 6],
	"c": range(2, 18, 4)
    }
)

df2 = pd.DataFrame(
    {
	"a": [5, 4, np.nan, 3, 7],
	"b": [np.nan, 3, 4, 6, 8]
    }
)

log("df1", df1)
log("df2", df2)
log("df1.combine_first(df2)", df1.combine_first(df2))

===================================== df1 ======================================
     a    b   c
0  1.0  NaN   2
1  NaN  2.0   6
2  5.0  NaN  10
3  NaN  6.0  14
===================================== df2 ======================================
     a    b
0  5.0  NaN
1  4.0  3.0
2  NaN  4.0
3  3.0  6.0
4  7.0  8.0
============================ df1.combine_first(df2) ============================
     a    b     c
0  1.0  NaN   2.0
1  4.0  2.0   6.0
2  5.0  4.0  10.0
3  3.0  6.0  14.0
4  7.0  8.0   NaN

• stack

  将列中的数据透视到行

• unstack

  将行中的数据透视到列

df = pd.DataFrame(np.arange(6).reshape((2, 3)),
		  index=pd.Index(['Ohio', 'Colorado'], name='state'),
		  columns=pd.Index(['one', 'two', 'three'], name='number')
)
log("df", df)
log("df.stack()", df.stack())
log("df.stack().unstack()", df.stack().unstack())
log("df.stack().unstack(0)", df.stack().unstack(0))
log("df.stack().unstack('state')", df.stack().unstack('state'))

====================================== df ======================================
number    one  two  three
state
Ohio        0    1      2
Colorado    3    4      5
================================== df.stack() ==================================
state     number
Ohio      one       0
          two       1
          three     2
Colorado  one       3
          two       4
          three     5
dtype: int64
============================= df.stack().unstack() =============================
number    one  two  three
state
Ohio        0    1      2
Colorado    3    4      5
============================ df.stack().unstack(0) =============================
state   Ohio  Colorado
number
one        0         3
two        1         4
three      2         5
========================= df.stack().unstack('state') ==========================
state   Ohio  Colorado
number
one        0         3
two        1         4
three      2         5

当层级中的所有值并未包含于每个子分组中， unstack 会引入缺失值：

s1 = pd.Series([0, 1, 2, 3], index=list('abcd'))
s2 = pd.Series([4, 5, 6], index=list('cde'))
df = pd.concat([s1, s2], keys=['one', 'two'])
log("df", df)
log("df.unstack()", df.unstack())

====================================== df ======================================
one  a    0
     b    1
     c    2
     d    3
two  c    4
     d    5
     e    6
dtype: int64
================================= df.unstack() =================================
       a    b    c    d    e
one  0.0  1.0  2.0  3.0  NaN
two  NaN  NaN  4.0  5.0  6.0

默认情况下， stack 会滤除缺失值：

log("df.unstack().stack()", df.unstack().stack())
log("df.unstack().stack(dropna=False)", df.unstack().stack(dropna=False))

============================= df.unstack().stack() =============================
one  a    0.0
     b    1.0
     c    2.0
     d    3.0
two  c    4.0
     d    5.0
     e    6.0
dtype: float64
======================= df.unstack().stack(dropna=False) =======================
one  a    0.0
     b    1.0
     c    2.0
     d    3.0
     e    NaN
two  a    NaN
     b    NaN
     c    4.0
     d    5.0
     e    6.0
dtype: float64

pivot(<用作行索引的列>, <用作列索引的列>, <可选的数值列>)

data = [
    ['1959-03-31', 'realgdp', 2710.349],
    ['1959-03-31', 'infl', 0],
    ['1959-03-31', 'unemp', 5.8],
    ['1959-06-30', 'realgdp', 2778.801],
    ['1959-06-30', 'infl', 2.34],
    ['1959-06-30', 'unemp', 5.1],
    ['1959-09-30', 'realgdp', 2775.488],
    ['1959-09-30', 'infl', 2.74],
    ['1959-09-30', 'unemp', 5.3],
    ['1959-12-31', 'realgdp', 2785.204]
]

df = pd.DataFrame(data, columns=['date', 'item', 'value'])
log("df", df)
# 针对所有 date ，将 item 的值转换为列索引，填入 value
log("df.pivot('date', 'item', 'value')", df.pivot('date', 'item', 'value'))

====================================== df ======================================
         date     item     value
0  1959-03-31  realgdp  2710.349
1  1959-03-31     infl     0.000
2  1959-03-31    unemp     5.800
3  1959-06-30  realgdp  2778.801
4  1959-06-30     infl     2.340
5  1959-06-30    unemp     5.100
6  1959-09-30  realgdp  2775.488
7  1959-09-30     infl     2.740
8  1959-09-30    unemp     5.300
9  1959-12-31  realgdp  2785.204
====================== df.pivot('date', 'item', 'value') =======================
item        infl   realgdp  unemp
date
1959-03-31  0.00  2710.349    5.8
1959-06-30  2.34  2778.801    5.1
1959-09-30  2.74  2775.488    5.3
1959-12-31   NaN  2785.204    NaN

多个数值列:

df['value2'] = np.random.randn(len(df))
log("df", df)
log("df.pivot('date', 'item')", df.pivot('date', 'item'))

====================================== df ======================================
         date     item     value    value2
0  1959-03-31  realgdp  2710.349 -0.894763
1  1959-03-31     infl     0.000  1.673363
2  1959-03-31    unemp     5.800  0.185811
3  1959-06-30  realgdp  2778.801 -0.905169
4  1959-06-30     infl     2.340  0.697258
5  1959-06-30    unemp     5.100  0.342515
6  1959-09-30  realgdp  2775.488 -0.544010
7  1959-09-30     infl     2.740  0.591888
8  1959-09-30    unemp     5.300 -0.605425
9  1959-12-31  realgdp  2785.204  1.386318
=========================== df.pivot('date', 'item') ===========================
           value                    value2
item        infl   realgdp unemp      infl   realgdp     unemp
date
1959-03-31  0.00  2710.349   5.8  1.673363 -0.894763  0.185811
1959-06-30  2.34  2778.801   5.1  0.697258 -0.905169  0.342515
1959-09-30  2.74  2775.488   5.3  0.591888 -0.544010 -0.605425
1959-12-31   NaN  2785.204   NaN       NaN  1.386318       NaN

注意：

pivot 等价于使用 set_index 创建分层索引，然后调用 unstack ：

log("df.set_index(['date', 'item']).unstack('item')", df.set_index(['date', 'item']).unstack('item'))

================ df.set_index(['date', 'item']).unstack('item') ================
           value                    value2
item        infl   realgdp unemp      infl   realgdp     unemp
date
1959-03-31  0.00  2710.349   5.8  1.673363 -0.894763  0.185811
1959-06-30  2.34  2778.801   5.1  0.697258 -0.905169  0.342515
1959-09-30  2.74  2775.488   5.3  0.591888 -0.544010 -0.605425
1959-12-31   NaN  2785.204   NaN       NaN  1.386318       NaN

pd.melt(df, <分组指标>)

melt 用于将多列合并成一列（ 每个观测值分别位于一行 ），可以看成是 pivot 的 反操作 。

df = pd.DataFrame(
    {
	"key": ['foo', 'bar', 'baz'],
	"A": [1, 2, 3],
	"B": [4, 5, 6],
	"C": [7, 8, 9]
    }
)
log("df", df)
# key 列作为分组指标，其他列均作为数据值
log("pd.melt(df, ['key'])", pd.melt(df, ['key']))

====================================== df ======================================
   key  A  B  C
0  foo  1  4  7
1  bar  2  5  8
2  baz  3  6  9
============================= pd.melt(df, ['key']) =============================
   key variable  value
0  foo        A      1
1  bar        A      2
2  baz        A      3
3  foo        B      4
4  bar        B      5
5  baz        B      6
6  foo        C      7
7  bar        C      8
8  baz        C      9

log("df", df)
log("df.groupby('key1').get_group('a')", df.groupby('key1').get_group('a'))

====================================== df ======================================
  key1 key2     data1     data2
0    a  one  0.560161  0.462387
1    a  two -0.815368 -0.866780
2    b  one  1.004914  0.653270
3    b  two  0.361801 -1.956438
4    a  one -0.340487 -0.128474
====================== df.groupby('key1').get_group('a') =======================
  key1 key2     data1     data2
0    a  one  0.560161  0.462387
1    a  two -0.815368 -0.866780
4    a  one -0.340487 -0.128474

对 GroupBy 对象进行列索引，等效于对该列或列的子集做聚合。

log("df", df)
log("list(df.groupby('key1')", list(df.groupby('key1')))
print()
# 等效于 df['date1'].groupby(df['key1'])
log("df.groupby('key1')['data1']", df.groupby('key1')['data1'])
log("list(df.groupby('key1')['data1'])", list(df.groupby('key1')['data1']))

# 等效于 df[['date2']].groupby(df['key1'])
log("df.groupby('key1')[['data2']]", df.groupby('key1')[['data2']])
log("list(df.groupby('key1')[['data2']])", list(df.groupby('key1')[['data2']]))
print()
log("type(df.groupby(['key1', 'key2'])[['data2']].mean())", type(df.groupby(['key1', 'key2'])[['data2']].mean()))
log("df.groupby(['key1', 'key2'])[['data2']].mean()", df.groupby(['key1', 'key2'])[['data2']].mean())

log("type(df.groupby(['key1', 'key2'])['data2'].mean())", type(df.groupby(['key1', 'key2'])['data2'].mean()))
log("df.groupby(['key1', 'key2'])['data2'].mean()", df.groupby(['key1', 'key2'])['data2'].mean())

====================================== df ======================================
  key1 key2     data1     data2
0    a  one -0.199651 -0.279757
1    a  two -1.053575  0.356160
2    b  one  2.185409  1.448519
3    b  two -0.701065  1.565013
4    a  one  0.666953  0.000710
=========================== list(df.groupby('key1') ============================
[('a',   key1 key2     data1     data2
0    a  one -0.199651 -0.279757
1    a  two -1.053575  0.356160
4    a  one  0.666953  0.000710), ('b',   key1 key2     data1     data2
2    b  one  2.185409  1.448519
3    b  two -0.701065  1.565013)]

========================= df.groupby('key1')['data1'] ==========================
<pandas.core.groupby.groupby.SeriesGroupBy object at 0x112278320>
====================== list(df.groupby('key1')['data1']) =======================
[('a', 0   -0.199651
1   -1.053575
4    0.666953
Name: data1, dtype: float64), ('b', 2    2.185409
3   -0.701065
Name: data1, dtype: float64)]
======================== df.groupby('key1')[['data2']] =========================
<pandas.core.groupby.groupby.DataFrameGroupBy object at 0x1122784a8>
===================== list(df.groupby('key1')[['data2']]) ======================
[('a',   key1 key2     data1     data2
0    a  one -0.199651 -0.279757
1    a  two -1.053575  0.356160
4    a  one  0.666953  0.000710), ('b',   key1 key2     data1     data2
2    b  one  2.185409  1.448519
3    b  two -0.701065  1.565013)]

============= type(df.groupby(['key1', 'key2'])[['data2']].mean()) =============
<class 'pandas.core.frame.DataFrame'>
================ df.groupby(['key1', 'key2'])[['data2']].mean() ================
              data2
key1 key2
a    one  -0.139523
     two   0.356160
b    one   1.448519
     two   1.565013
============== type(df.groupby(['key1', 'key2'])['data2'].mean()) ==============
<class 'pandas.core.series.Series'>
================= df.groupby(['key1', 'key2'])['data2'].mean() =================
key1  key2
a     one    -0.139523
      two     0.356160
b     one     1.448519
      two     1.565013
Name: data2, dtype: float64

people = pd.DataFrame(np.random.randn(5, 5),
		      columns=list('abcde'),
		      index=['Joe', 'Steve', 'Wes', 'Jim', 'Travis']
)
people.iloc[2:3, [1, 2]] = np.nan
log("people", people)
mapping = {
    "a": 'red',
    "b": 'red',
    "c": 'blue',
    "d": 'blue',
    "e": 'red',
    "f": 'orange'               # 没有作用
}
log("people.groupby(mapping, axis=1).sum()", people.groupby(mapping, axis=1).sum())

==================================== people ====================================
               a         b         c         d         e
Joe     0.411963 -0.851885 -0.603098  0.711264  0.244195
Steve  -0.192852 -0.761196  0.147913 -1.878708 -1.548063
Wes     0.226710       NaN       NaN -1.139280 -0.126362
Jim    -1.152243 -0.869533 -0.078809 -0.971286 -1.517877
Travis  0.381503  0.533020  0.446621  1.446923 -0.279701
==================== people.groupby(mapping, axis=1).sum() =====================
            blue       red
Joe     0.108166 -0.195727
Steve  -1.730795 -2.502111
Wes    -1.139280  0.100347
Jim    -1.050096 -3.539654
Travis  1.893544  0.634822

map_series = pd.Series(mapping)
log("map_series", map_series)
log("people.groupby(map_series, axis=1).count()", people.groupby(map_series, axis=1).count())

================================== map_series ==================================
a       red
b       red
c      blue
d      blue
e       red
f    orange
dtype: object
================== people.groupby(map_series, axis=1).count() ==================
        blue  red
Joe        2    3
Steve      2    3
Wes        1    2
Jim        2    3
Travis     2    3

使用函数定义分组关系是一种更为通用的方式， 作为分组键传递的函数将会按照每个索引值调用一次，返回值会被用作分组名称。

log("people", people)
log("people.groupby(len).sum()", people.groupby(len).sum())

==================================== people ====================================
               a         b         c         d         e
Joe     0.411963 -0.851885 -0.603098  0.711264  0.244195
Steve  -0.192852 -0.761196  0.147913 -1.878708 -1.548063
Wes     0.226710       NaN       NaN -1.139280 -0.126362
Jim    -1.152243 -0.869533 -0.078809 -0.971286 -1.517877
Travis  0.381503  0.533020  0.446621  1.446923 -0.279701
========================== people.groupby(len).sum() ===========================
          a         b         c         d         e
3 -0.513571 -1.721418 -0.681908 -1.399303 -1.400044
5 -0.192852 -0.761196  0.147913 -1.878708 -1.548063
6  0.381503  0.533020  0.446621  1.446923 -0.279701

能够在轴索引的某个层级上进行聚合。

将层级数值或名称传递给 level 关键字：

columns = pd.MultiIndex.from_arrays(
    [
	['US', 'US', 'US', 'JP', 'JP'],
	[1, 3, 5, 1, 3]
    ],
    names=['cty', 'tenor'])
df = pd.DataFrame(np.random.randn(4, 5), columns=columns)
log("df", df)
log("df.groupby(level='cty', axis=1).count())", df.groupby(level='cty', axis=1).count())

====================================== df ======================================
cty          US                            JP
tenor         1         3         5         1         3
0      1.102170 -0.112435  1.085613 -0.967759  0.883713
1      0.076930 -0.051798 -0.141812  0.625950  0.097327
2     -0.261993 -0.041604  0.793321 -1.847407 -1.173491
3     -2.023053  0.002613 -0.638650 -0.320731 -0.519337
=================== df.groupby(level='cty', axis=1).count()) ===================
cty  JP  US
0     2   3
1     2   3
2     2   3
3     2   3

df = pd.DataFrame({'key1' : ['a', 'a', 'b', 'b', 'a'],
		   'key2' : ['one', 'two', 'one', 'two', 'one'],
		   'data1' : np.random.randn(5),
		   'data2' : np.random.randn(5)})
log("df", df)
log("df.groupby('key1')['data1'].quantile(0.9)", df.groupby('key1')['data1'].quantile(0.9))
log("df['data1'].groupby(df['key1']).quantile(0.9)", df['data1'].groupby(df['key1']).quantile(0.9))

def my_quantile_0_9(s):
    return s.quantile(0.9)

log("df.groupby('key1')['data1'].agg(my_quantile_0_9)", df.groupby('key1')['data1'].agg(my_quantile_0_9))
log("df.groupby('key1').agg(my_quantile_0_9)", df.groupby('key1').agg(my_quantile_0_9))

====================================== df ======================================
  key1 key2     data1     data2
0    a  one  0.115562 -0.159065
1    a  two -0.953228 -0.255576
2    b  one -0.552461 -0.230414
3    b  two  1.400817  0.179607
4    a  one -0.337459  0.781640
================== df.groupby('key1')['data1'].quantile(0.9) ===================
key1
a    0.024958
b    1.205490
Name: data1, dtype: float64
================ df['data1'].groupby(df['key1']).quantile(0.9) =================
key1
a    0.024958
b    1.205490
Name: data1, dtype: float64
=============== df.groupby('key1')['data1'].agg(my_quantile_0_9) ===============
key1
a    0.024958
b    1.205490
Name: data1, dtype: float64
=================== df.groupby('key1').agg(my_quantile_0_9) ====================
         data1     data2
key1
a     0.024958  0.593499
b     1.205490  0.138605

通过向 groupby 传递 as_index = False 来禁用分组键作为索引的行为： （通过在结果上调用 reset_index 可以获得相同的效果）

log("tips.groupby(['day', 'smoker'], as_index=False).mean()", tips.groupby(['day', 'smoker'], as_index=False).mean())

================= tips.groupby(['day', 'smoker'], as_index=False).mean() =================
   day smoker  total_bill   tip      size   tip_pct
0  Fri    Yes      25.635  3.46  3.000000  0.135312
1  Sat     No      17.490  1.01  2.000000  0.057795
2  Sun    Yes      18.880  3.29  3.333333  0.171123

GroupBy.agg 方法将一个函数使用在一个数列上，然后返回一个标量的值。即每次传入的是一列数据，对其聚合后返回标量。

GroupBy.apply 是一个 更一般化 的方法：将一个数据分拆-应用-汇总。会将当前分组后的数据一起传入，可以返回标量，也可以返回多维数据。

在 GroupBy 对象内部，当调用 describe 这样的方法时，实际上是以下代码的 简写 ：

f = lambda x: x.describe()
grouped.apply(f)

df = pd.DataFrame({'key1': ['a', 'a', 'b', 'b', 'a', 'a', 'a', 'b', 'b', 'a'],
		  'key2': ['one', 'two', 'one', 'two', 'one', 'one', 'two', 'one', 'two', 'one'],
		  'data1': np.random.randint(1, 10, 10),
		  'data2': np.random.randint(1, 10, 10)})

log("df", df)
# 根据 column 排序，输出其最大的 n 行数据
def top(df, n=2, column='data1'):
    return df.sort_values(by=column, ascending=False)[:n]

log("df.groupby('key1').apply(top)", df.groupby('key1').apply(top))
# 可以传递参数：df.groupby('key1').apply(top, n=3, column='data2')

=========================================== df ===========================================
  key1 key2  data1  data2
0    a  one      2      3
1    a  two      5      4
2    b  one      2      4
3    b  two      7      7
4    a  one      4      9
5    a  one      2      6
6    a  two      6      3
7    b  one      2      8
8    b  two      9      1
9    a  one      8      2
============================= df.groupby('key1').apply(top) ==============================
       key1 key2  data1  data2
key1
a    9    a  one      8      2
     6    a  two      6      3
b    8    b  two      9      1
     3    b  two      7      7

当返回结果集中不需要体现分组信息时，可以使用该参数。

df = pd.DataFrame({'key1': ['a', 'a', 'b', 'b', 'a', 'a', 'a', 'b', 'b', 'a'],
		    'key2': ['one', 'two', 'one', 'two', 'one', 'one', 'two', 'one', 'two', 'one'],
		    'data1': np.random.randint(1, 10, 10),
		    'data2': np.random.randint(1, 10, 10)})

log("df", df)

# 根据 column 排序，输出其最大的 n 行数据
def top(df, n=2, column='data1'):
    return df.sort_values(by=column, ascending=False)[:n]

log("df.groupby('key1').apply(top)", df.groupby('key1').apply(top))
log("df.groupby('key1', group_keys=False).apply(top)", df.groupby('key1', group_keys=False).apply(top))

=========================================== df ===========================================
  key1 key2  data1  data2
0    a  one      8      9
1    a  two      4      1
2    b  one      2      6
3    b  two      5      6
4    a  one      6      5
5    a  one      4      8
6    a  two      7      2
7    b  one      6      9
8    b  two      7      1
9    a  one      3      5
============================= df.groupby('key1').apply(top) ==============================
       key1 key2  data1  data2
key1
a    0    a  one      8      9
     6    a  two      7      2
b    8    b  two      7      1
     7    b  one      6      9
==================== df.groupby('key1', group_keys=False).apply(top) =====================
  key1 key2  data1  data2
0    a  one      8      9
6    a  two      7      2
8    b  two      7      1
7    b  one      6      9

GroupBy.transform 传递一个函数 f(x) ， x 根据 GroupBy 对象的具体类型可以是 Series ，也可以是 DataFrame 。

f(x) 的返回值可以是标量（ 广播 到各分组的数据中），也可以是与输入 尺寸相同 的对象。

对时间序列数据而言， resample 方法从语义上讲可以认为也是分组操作。

N=15
times = pd.date_range('2017-05-20 00:00', freq='1min', periods=N)
df = pd.DataFrame({'time': times,
		   'value': np.arange(N)})
log("df", df)
log("df.set_index('time').resample('5min').count()", df.set_index('time').resample('5min').count())

================================================ df ================================================
                  time  value
0  2017-05-20 00:00:00      0
1  2017-05-20 00:01:00      1
2  2017-05-20 00:02:00      2
3  2017-05-20 00:03:00      3
4  2017-05-20 00:04:00      4
5  2017-05-20 00:05:00      5
6  2017-05-20 00:06:00      6
7  2017-05-20 00:07:00      7
8  2017-05-20 00:08:00      8
9  2017-05-20 00:09:00      9
10 2017-05-20 00:10:00     10
11 2017-05-20 00:11:00     11
12 2017-05-20 00:12:00     12
13 2017-05-20 00:13:00     13
14 2017-05-20 00:14:00     14
========================== df.set_index('time').resample('5min').count() ===========================
                     value
time
2017-05-20 00:00:00      5
2017-05-20 00:05:00      5
2017-05-20 00:10:00      5

my_categories = pd.Categorical(['foo', 'bar', 'baz', 'foo', 'bar'])
log("my_categories", my_categories)

categories = ['foo', 'bar', 'baz']
codes=[0,1,2,0,0,1]
my_cats_2 = pd.Categorical.from_codes(codes, categories)
log("my_cats_2", my_cats_2)

========================================== my_categories ===========================================
[foo, bar, baz, foo, bar]
Categories (3, object): [bar, baz, foo]
============================================ my_cats_2 =============================================
[foo, bar, baz, foo, foo, bar]
Categories (3, object): [foo, bar, baz]

ordered_cat = pd.Categorical.from_codes(codes, categories, ordered=True)
log("ordered_cat", ordered_cat)
log("ordered_cat.sort_values()", ordered_cat.sort_values())

# 将无序对象转换为有序
log("my_cats_2.as_ordered()", my_cats_2.as_ordered())

=========================================== ordered_cat ============================================
[foo, bar, baz, foo, foo, bar]
Categories (3, object): [foo < bar < baz]
==================================== ordered_cat.sort_values() =====================================
[foo, foo, foo, bar, bar, baz]
Categories (3, object): [foo < bar < baz]
====================================== my_cats_2.as_ordered() ======================================
[foo, bar, baz, foo, foo, bar]
Categories (3, object): [foo < bar < baz]

np.random.seed(12345)
draws = np.random.randn(1000)
log("draws[:5]", draws[:5])

bins = pd.qcut(draws, 4)
log("bins", bins)

# 用标签替代区间，增加可读性
labelBins = pd.qcut(draws, 4, labels=['Q1', 'Q2', 'Q3', 'Q4'])
log("labelBins", labelBins)

# 使用 groupby 提取统计信息
labelBinsSeries = pd.Series(labelBins, name='quartile')
log("pd.Series(draws).groupby(labelBinsSeries).agg(['count', 'min', 'max']).reset_index()", pd.Series(draws).groupby(labelBinsSeries).agg(['count', 'min', 'max']).reset_index())

============================================ draws[:5] =============================================
[-0.20470766  0.47894334 -0.51943872 -0.5557303   1.96578057]
=============================================== bins ===============================================
[(-0.684, -0.0101], (-0.0101, 0.63], (-0.684, -0.0101], (-0.684, -0.0101], (0.63, 3.928], ..., (-0.0101, 0.63], (-0.684, -0.0101], (-2.9499999999999997, -0.684], (-0.0101, 0.63], (0.63, 3.928]]
Length: 1000
Categories (4, interval[float64]): [(-2.9499999999999997, -0.684] < (-0.684, -0.0101] < (-0.0101, 0.63] < (0.63, 3.928]]
============================================ labelBins =============================================
[Q2, Q3, Q2, Q2, Q4, ..., Q3, Q2, Q1, Q3, Q4]
Length: 1000
Categories (4, object): [Q1 < Q2 < Q3 < Q4]
======= pd.Series(draws).groupby(labelBinsSeries).agg(['count', 'min', 'max']).reset_index() =======
  quartile  count       min       max
0       Q1    250 -2.949343 -0.685484
1       Q2    250 -0.683066 -0.010115
2       Q3    250 -0.010032  0.628894
3       Q4    250  0.634238  3.927528

N = 10000000
labels = pd.Series(['foo', 'bar', 'baz', 'qux'] * (N // 4))
categories = labels.astype('category')
log("labels.memory_usage()", labels.memory_usage())
log("categories.memory_usage()", categories.memory_usage())

====================================== labels.memory_usage() =======================================
80000080
==================================== categories.memory_usage() =====================================
10000272

内存性能的提升是以计算耗时换来的：

%time labels.astype('category')

CPU times: user 470 ms, sys: 50.2 ms, total: 521 ms
Wall time: 518 ms

from datetime import datetime
dates = [datetime(2011, 1, 2), datetime(2011, 1, 5),
	 datetime(2011, 1, 7), datetime(2011, 1, 8),
	 datetime(2011, 1, 10), datetime(2011, 1, 12)]
ts = pd.Series(np.random.randn(6), index=dates)
log("ts", ts)

====================================== ts ======================================
2011-01-02    0.488675
2011-01-05   -0.178098
2011-01-07    2.122315
2011-01-08    0.061192
2011-01-10    0.884111
2011-01-12   -0.608506
dtype: float64

log("ts['1/10/2011']", ts['1/10/2011'])
log("ts['20110110']", ts['20110110'])

=============================== ts['1/10/2011'] ================================
0.8841108212188903
================================ ts['20110110'] ================================
0.8841108212188903

longer_ts = pd.Series(np.random.randn(1000),
		      index=pd.date_range('1/1/2000', periods=1000))
log("longer_ts.head()", longer_ts.head())

=============================== longer_ts.head() ===============================
2000-01-01   -0.072052
2000-01-02    0.544066
2000-01-03    0.323886
2000-01-04   -1.683325
2000-01-05    0.526860
Freq: D, dtype: float64

log("longer_ts['2001'].head()", longer_ts['2001'].head())
log("longer_ts['2001-05'].head()", longer_ts['2001-05'].head())
log("ts[datetime(2011, 1, 7):]", ts[datetime(2011, 1, 7):])
# 可以使用不包含在时间序列中的时间戳进行切片
log("ts['1/6/2011':'1/11/2011']", ts['1/6/2011':'1/11/2011'])

=========================== longer_ts['2001'].head() ===========================
2001-01-01   -0.171724
2001-01-02   -0.827840
2001-01-03    0.370548
2001-01-04   -0.674167
2001-01-05   -1.712340
Freq: D, dtype: float64
========================= longer_ts['2001-05'].head() ==========================
2001-05-01    0.251343
2001-05-02    1.624623
2001-05-03   -0.240567
2001-05-04    0.163052
2001-05-05   -0.586669
Freq: D, dtype: float64
========================== ts[datetime(2011, 1, 7):] ===========================
2011-01-07    2.122315
2011-01-08    0.061192
2011-01-10    0.884111
2011-01-12   -0.608506
dtype: float64
========================== ts['1/6/2011':'1/11/2011'] ==========================
2011-01-07    2.122315
2011-01-08    0.061192
2011-01-10    0.884111
dtype: float64

log("pd.date_range('2012-04-01', '2012-04-20')", pd.date_range('2012-04-01', '2012-04-20'))
log("pd.date_range(start='2012-04-01', periods=20)", pd.date_range(start='2012-04-01', periods=20))

================== pd.date_range('2012-04-01', '2012-04-20') ===================
DatetimeIndex(['2012-04-01', '2012-04-02', '2012-04-03', '2012-04-04',
               '2012-04-05', '2012-04-06', '2012-04-07', '2012-04-08',
               '2012-04-09', '2012-04-10', '2012-04-11', '2012-04-12',
               '2012-04-13', '2012-04-14', '2012-04-15', '2012-04-16',
               '2012-04-17', '2012-04-18', '2012-04-19', '2012-04-20'],
              dtype='datetime64[ns]', freq='D')
================ pd.date_range(start='2012-04-01', periods=20) =================
DatetimeIndex(['2012-04-01', '2012-04-02', '2012-04-03', '2012-04-04',
               '2012-04-05', '2012-04-06', '2012-04-07', '2012-04-08',
               '2012-04-09', '2012-04-10', '2012-04-11', '2012-04-12',
               '2012-04-13', '2012-04-14', '2012-04-15', '2012-04-16',
               '2012-04-17', '2012-04-18', '2012-04-19', '2012-04-20'],
              dtype='datetime64[ns]', freq='D')
============= pd.date_range('2000-01-01', '2000-12-01', freq='BM') =============
DatetimeIndex(['2000-01-31', '2000-02-29', '2000-03-31', '2000-04-28',
               '2000-05-31', '2000-06-30', '2000-07-31', '2000-08-31',
               '2000-09-29', '2000-10-31', '2000-11-30'],
              dtype='datetime64[ns]', freq='BM')

log("pd.date_range('2000-01-01', '2000-12-01', freq='BM')", pd.date_range('2000-01-01', '2000-12-01', freq='BM'))
log("pd.date_range('2000-01-01', periods=10, freq='1h30min')", pd.date_range('2000-01-01', periods=10, freq='1h30min'))

============= pd.date_range('2000-01-01', '2000-12-01', freq='BM') =============
DatetimeIndex(['2000-01-31', '2000-02-29', '2000-03-31', '2000-04-28',
               '2000-05-31', '2000-06-30', '2000-07-31', '2000-08-31',
               '2000-09-29', '2000-10-31', '2000-11-30'],
              dtype='datetime64[ns]', freq='BM')
=========== pd.date_range('2000-01-01', periods=10, freq='1h30min') ============
DatetimeIndex(['2000-01-01 00:00:00', '2000-01-01 01:30:00',
               '2000-01-01 03:00:00', '2000-01-01 04:30:00',
               '2000-01-01 06:00:00', '2000-01-01 07:30:00',
               '2000-01-01 09:00:00', '2000-01-01 10:30:00',
               '2000-01-01 12:00:00', '2000-01-01 13:30:00'],
              dtype='datetime64[ns]', freq='90T')

frame = pd.DataFrame(np.random.randn(2, 4),
		     index=pd.date_range('1/1/2000', periods=2, freq='W-WED'),
		     columns=['Colorado', 'Texas', 'New York', 'Ohio'])
log("frame", frame)
log("frame.resample('D').asfreq()", frame.resample('D').asfreq())
log("frame.resample('D').ffill()", frame.resample('D').ffill())

==================================== frame =====================================
            Colorado     Texas  New York      Ohio
2000-01-05 -0.179199  0.929787 -0.046577  0.028099
2000-01-12 -0.196553  0.334500  0.171028  1.063931
========================= frame.resample('D').asfreq() =========================
            Colorado     Texas  New York      Ohio
2000-01-05 -0.179199  0.929787 -0.046577  0.028099
2000-01-06       NaN       NaN       NaN       NaN
2000-01-07       NaN       NaN       NaN       NaN
2000-01-08       NaN       NaN       NaN       NaN
2000-01-09       NaN       NaN       NaN       NaN
2000-01-10       NaN       NaN       NaN       NaN
2000-01-11       NaN       NaN       NaN       NaN
2000-01-12 -0.196553  0.334500  0.171028  1.063931
========================= frame.resample('D').ffill() ==========================
            Colorado     Texas  New York      Ohio
2000-01-05 -0.179199  0.929787 -0.046577  0.028099
2000-01-06 -0.179199  0.929787 -0.046577  0.028099
2000-01-07 -0.179199  0.929787 -0.046577  0.028099
2000-01-08 -0.179199  0.929787 -0.046577  0.028099
2000-01-09 -0.179199  0.929787 -0.046577  0.028099
2000-01-10 -0.179199  0.929787 -0.046577  0.028099
2000-01-11 -0.179199  0.929787 -0.046577  0.028099
2000-01-12 -0.196553  0.334500  0.171028  1.063931

s = pd.Series(range(10))
log("s", s)
log("s.rolling(3).mean()", s.rolling(3).mean())
log("s.rolling(3, min_periods=2).mean()", s.rolling(3, min_periods=2).mean())
s.rolling(3).mean().plot()
plot()

====================================== s =======================================
0    0
1    1
2    2
3    3
4    4
5    5
6    6
7    7
8    8
9    9
dtype: int64
============================= s.rolling(3).mean() ==============================
0    NaN
1    NaN
2    1.0
3    2.0
4    3.0
5    4.0
6    5.0
7    6.0
8    7.0
9    8.0
dtype: float64
====================== s.rolling(3, min_periods=2).mean() ======================
0    NaN
1    0.5
2    1.0
3    2.0
4    3.0
5    4.0
6    5.0
7    6.0
8    7.0
9    8.0
dtype: float64

close_px_all = pd.read_csv('data/stock_px_2.csv', parse_dates=True, index_col=0)
close_px = close_px_all[['AAPL', 'MSFT', 'XOM']]
log("close_px.head()", close_px.head())
log("close_px.rolling('20D').mean().head()", close_px.rolling('20D').mean().head()) # 无需指定 min_periods ？

=============================== close_px.head() ================================
            AAPL   MSFT    XOM
2003-01-02  7.40  21.11  29.22
2003-01-03  7.45  21.14  29.24
2003-01-06  7.45  21.52  29.96
2003-01-07  7.43  21.93  28.95
2003-01-08  7.28  21.31  28.83
==================== close_px.rolling('20D').mean().head() =====================
                AAPL       MSFT        XOM
2003-01-02  7.400000  21.110000  29.220000
2003-01-03  7.425000  21.125000  29.230000
2003-01-06  7.433333  21.256667  29.473333
2003-01-07  7.432500  21.425000  29.342500
2003-01-08  7.402000  21.402000  29.240000

s = pd.Series(range(10))
log("s", s)
log("s.corr(s)", s.corr(s))
log("s.rolling(3).corr(s)", s.rolling(3).corr(s))

====================================== s =======================================
0    0
1    1
2    2
3    3
4    4
5    5
6    6
7    7
8    8
9    9
dtype: int64
================================== s.corr(s) ===================================
0.9999999999999999
============================= s.rolling(3).corr(s) =============================
0    NaN
1    1.0
2    1.0
3    1.0
4    1.0
5    1.0
6    1.0
7    1.0
8    1.0
9    1.0
dtype: float64

assign 方法是 df[k] = v 的链式表达方式。

df2 = df.copy()
df['k'] = v

等效于：

df2 = df.assign(k=v)

df = load_data()
df2 = df[df['col2'] < 0]
df2['col1_demeaned'] = df2.col1 - df2.col1.mean()

等效于：

df2 = (load_data()
       [lambda x: x.col2 < 0]
       .assign(col1_demeaned=lambda x: x.col1 - x.col1.mean()))

这种方式就不需要引用中间变量 df 了。

当使用函数处理 Series 或 DataFrame ，处理结果也是 Series 或 DataFrame 类型时， 可以使用 pipe 完成链式操作：

a = f(df, arg1=v1)
b = g(a, v2, arg3=v3)
c = h(b, arg4=v4)

等效于：

result = (df.pipe(f, arg1=v1)
	  .pipe(g, v2, arg3=v3)
	  .pipe(h, arg4=v4))

Series.searchsorted(value)

序列中的元素从小到大排列，提取其对应的 index

log("np.array([1,4,3,-1,6,9]).argsort()", np.array([1,4,3,-1,6,9]).argsort())
log("pd.Series(np.array([1,4,3,-1,6,9])).argsort()", pd.Series(np.array([1,4,3,-1,6,9])).argsort())

====================== np.array([1,4,3,-1,6,9]).argsort() ======================
[3 0 2 1 4 5]
================ pd.Series(np.array([1,4,3,-1,6,9])).argsort() =================
0    3
1    0
2    2
3    1
4    4
5    5
dtype: int64

df = pd.DataFrame(np.arange(12).reshape((3, 4)),
		  columns=pd.Index(list('abcd')))
log("df", df)
log("df.sum(0)", df.sum(0))
log("df.sum(1)", df.sum(1))
log("df.div(df.sum(0))", df.div(df.sum(0)))
log("df.div(df.sum(1), axis=0)", df.div(df.sum(1), axis=0))

====================================== df ======================================
   a  b   c   d
0  0  1   2   3
1  4  5   6   7
2  8  9  10  11
================================== df.sum(0) ===================================
a    12
b    15
c    18
d    21
dtype: int64
================================== df.sum(1) ===================================
0     6
1    22
2    38
dtype: int64
============================== df.div(df.sum(0)) ===============================
          a         b         c         d
0  0.000000  0.066667  0.111111  0.142857
1  0.333333  0.333333  0.333333  0.333333
2  0.666667  0.600000  0.555556  0.523810
========================== df.div(df.sum(1), axis=0) ===========================
          a         b         c         d
0  0.000000  0.166667  0.333333  0.500000
1  0.181818  0.227273  0.272727  0.318182
2  0.210526  0.236842  0.263158  0.289474

s = pd.Series([0, 0, 1, 1, 1, 2, 2, 2, 3, 3])
log("s", s)
log("s[s.shift() != s]", s[s.shift() != s])
log("s[s.shift(-1) != s]", s[s.shift(-1) != s])

====================================== s =======================================
0    0
1    0
2    1
3    1
4    1
5    2
6    2
7    2
8    3
9    3
dtype: int64
============================== s[s.shift() != s] ===============================
0    0
2    1
5    2
8    3
dtype: int64
============================= s[s.shift(-1) != s] ==============================
1    0
4    1
7    2
9    3
dtype: int64

用于计算数据之间的差值

data = {
    'open' : pd.Series([1, 2, 3, 4], index=pd.date_range('2018-10-04', periods=4))
}
df = pd.DataFrame(data)
log("df", df)
log("df.assign(diff=df.open.diff(-1))", df.assign(diff=df.open.diff(-1)))
log("df.assign(diff=df.open.diff(1))", df.assign(diff=df.open.diff(1)))

====================================== df ======================================
            open
2018-10-04     1
2018-10-05     2
2018-10-06     3
2018-10-07     4
======================= df.assign(diff=df.open.diff(-1)) =======================
            open  diff
2018-10-04     1  -1.0
2018-10-05     2  -1.0
2018-10-06     3  -1.0
2018-10-07     4   NaN
======================= df.assign(diff=df.open.diff(1)) ========================
            open  diff
2018-10-04     1   NaN
2018-10-05     2   1.0
2018-10-06     3   1.0
2018-10-07     4   1.0

data = {
    'open' : pd.Series([1, 2, 3, 4], index=pd.date_range('2018-10-04', periods=4))
}
df = pd.DataFrame(data)
log("df", df)
log("df.assign(pct=df.open.pct_change(1))", df.assign(pct=df.open.pct_change(1)))

====================================== df ======================================
            open
2018-10-04     1
2018-10-05     2
2018-10-06     3
2018-10-07     4
===================== df.assign(pct=df.open.pct_change(1)) =====================
            open       pct
2018-10-04     1       NaN
2018-10-05     2  1.000000
2018-10-06     3  0.500000
2018-10-07     4  0.333333

data = {
    'open' : pd.Series([1, 2, 3, 4], index=pd.date_range('2018-10-04', periods=4))
}
df = pd.DataFrame(data)
log("df", df)
log("df.open.cumsum()", df.open.cumsum())

====================================== df ======================================
            open
2018-10-04     1
2018-10-05     2
2018-10-06     3
2018-10-07     4
=============================== df.open.cumsum() ===============================
2018-10-04     1
2018-10-05     3
2018-10-06     6
2018-10-07    10
Freq: D, Name: open, dtype: int64

data = {
    'open' : pd.Series([5, 5.2, 8.5, 8], index=pd.date_range('2018-10-04', periods=4))
}
df = pd.DataFrame(data)
log("df", df)
df['pct'] = df.open.pct_change(1)
log("(df['pct'] + 1.0).cumprod()", (df['pct'] + 1.0).cumprod()) # 假设初始资金为 1 元

====================================== df ======================================
            open
2018-10-04   5.0
2018-10-05   5.2
2018-10-06   8.5
2018-10-07   8.0
========================= (df['pct'] + 1.0).cumprod() ==========================
2018-10-04     NaN
2018-10-05    1.04
2018-10-06    1.70
2018-10-07    1.60
Freq: D, Name: pct, dtype: float64

s = pd.Series(range(10))
s2 = np.where(s % 2 == 0, 'Even', 'Odd')
s3 = np.where(s > 5, 'Bigger', 'Smaller')
log("s2", s2)
log("s3", s3)

====================================== s2 ======================================
['Even' 'Odd' 'Even' 'Odd' 'Even' 'Odd' 'Even' 'Odd' 'Even' 'Odd']
====================================== s3 ======================================
['Smaller' 'Smaller' 'Smaller' 'Smaller' 'Smaller' 'Smaller' 'Bigger'
 'Bigger' 'Bigger' 'Bigger']