How to preview a part of a large pandas DataFrame, in iPython notebook?
df.head(5) # will print out the first 5 rows
df.tail(5) # will print out the 5 last rows
In this case, where the DataFrame is long but not too wide, you can simply slice it:
>>> df = pd.DataFrame({"A": range(1000), "B": range(1000)})
>>> df
<class 'pandas.core.frame.DataFrame'>
Int64Index: 1000 entries, 0 to 999
Data columns:
A 1000 non-null values
B 1000 non-null values
dtypes: int64(2)
>>> df[:5]
A B
0 0 0
1 1 1
2 2 2
3 3 3
4 4 4
ix is deprecated.
If it's both wide and long, I tend to use .ix:
>>> df = pd.DataFrame({i: range(1000) for i in range(100)})
>>> df.ix[:5, :10]
0 1 2 3 4 5 6 7 8 9 10
0 0 0 0 0 0 0 0 0 0 0 0
1 1 1 1 1 1 1 1 1 1 1 1
2 2 2 2 2 2 2 2 2 2 2 2
3 3 3 3 3 3 3 3 3 3 3 3
4 4 4 4 4 4 4 4 4 4 4 4
5 5 5 5 5 5 5 5 5 5 5 5
I write a method to show the four corners of the data and monkey-patch to dataframe to do so:
def _sw(df, up_rows=10, down_rows=5, left_cols=4, right_cols=3, return_df=False):
''' display df data at four corners
A,B (up_pt)
C,D (down_pt)
parameters : up_rows=10, down_rows=5, left_cols=4, right_cols=3
usage:
df = pd.DataFrame(np.random.randn(20,10), columns=list('ABCDEFGHIJKLMN')[0:10])
df.sw(5,2,3,2)
df1 = df.set_index(['A','B'], drop=True, inplace=False)
df1.sw(5,2,3,2)
'''
#pd.set_printoptions(max_columns = 80, max_rows = 40)
ncol, nrow = len(df.columns), len(df)
# handle columns
if ncol <= (left_cols + right_cols) :
up_pt = df.ix[0:up_rows, :] # screen width can contain all columns
down_pt = df.ix[-down_rows:, :]
else: # screen width can not contain all columns
pt_a = df.ix[0:up_rows, 0:left_cols]
pt_b = df.ix[0:up_rows, -right_cols:]
pt_c = df[-down_rows:].ix[:,0:left_cols]
pt_d = df[-down_rows:].ix[:,-right_cols:]
up_pt = pt_a.join(pt_b, how='inner')
down_pt = pt_c.join(pt_d, how='inner')
up_pt.insert(left_cols, '..', '..')
down_pt.insert(left_cols, '..', '..')
overlap_qty = len(up_pt) + len(down_pt) - len(df)
down_pt = down_pt.drop(down_pt.index[range(overlap_qty)]) # remove overlap rows
dt_str_list = down_pt.to_string().split('\n') # transfer down_pt to string list
# Display up part data
print up_pt
start_row = (1 if df.index.names[0] is None else 2) # start from 1 if without index
# Display omit line if screen height is not enought to display all rows
if overlap_qty < 0:
print "." * len(dt_str_list[start_row])
# Display down part data row by row
for line in dt_str_list[start_row:]:
print line
# Display foot note
print "\n"
print "Index :",df.index.names
print "Column:",",".join(list(df.columns.values))
print "row: %d col: %d"%(len(df), len(df.columns))
print "\n"
return (df if return_df else None)
DataFrame.sw = _sw #add a method to DataFrame class
Here is the sample:
>>> df = pd.DataFrame(np.random.randn(20,10), columns=list('ABCDEFGHIJKLMN')[0:10])
>>> df.sw()
A B C D .. H I J
0 -0.8166 0.0102 0.0215 -0.0307 .. -0.0820 1.2727 0.6395
1 1.0659 -1.0102 -1.3960 0.4700 .. 1.0999 1.1222 -1.2476
2 0.4347 1.5423 0.5710 -0.5439 .. 0.2491 -0.0725 2.0645
3 -1.5952 -1.4959 2.2697 -1.1004 .. -1.9614 0.6488 -0.6190
4 -1.4426 -0.8622 0.0942 -0.1977 .. -0.7802 -1.1774 1.9682
5 1.2526 -0.2694 0.4841 -0.7568 .. 0.2481 0.3608 -0.7342
6 0.2108 2.5181 1.3631 0.4375 .. -0.1266 1.0572 0.3654
7 -1.0617 -0.4743 -1.7399 -1.4123 .. -1.0398 -1.4703 -0.9466
8 -0.5682 -1.3323 -0.6992 1.7737 .. 0.6152 0.9269 2.1854
9 0.2361 0.4873 -1.1278 -0.2251 .. 1.4232 2.1212 2.9180
10 2.0034 0.5454 -2.6337 0.1556 .. 0.0016 -1.6128 -0.8093
..............................................................
15 1.4091 0.3540 -1.3498 -1.0490 .. 0.9328 0.3668 1.3948
16 0.4528 -0.3183 0.4308 -0.1818 .. 0.1295 1.2268 0.1365
17 -0.7093 1.3991 0.9501 2.1227 .. -1.5296 1.1908 0.0318
18 1.7101 0.5962 0.8948 1.5606 .. -0.6862 0.9558 -0.5514
19 1.0329 -1.2308 -0.6896 -0.5112 .. 0.2719 1.1478 -0.1459
Index : [None]
Column: A,B,C,D,E,F,G,H,I,J
row: 20 col: 10
>>> df.sw(4,2,3,4)
A B C .. G H I J
0 -0.8166 0.0102 0.0215 .. 0.3671 -0.0820 1.2727 0.6395
1 1.0659 -1.0102 -1.3960 .. 1.0984 1.0999 1.1222 -1.2476
2 0.4347 1.5423 0.5710 .. 1.6675 0.2491 -0.0725 2.0645
3 -1.5952 -1.4959 2.2697 .. 0.4856 -1.9614 0.6488 -0.6190
4 -1.4426 -0.8622 0.0942 .. -0.0947 -0.7802 -1.1774 1.9682
..............................................................
18 1.7101 0.5962 0.8948 .. -0.8592 -0.6862 0.9558 -0.5514
19 1.0329 -1.2308 -0.6896 .. -0.3954 0.2719 1.1478 -0.1459
Index : [None]
Column: A,B,C,D,E,F,G,H,I,J
row: 20 col: 10
Here's a quick way to preview a large table without having it run too wide:
Display function:
# display large dataframes in an html iframe
def ldf_display(df, lines=500):
txt = ("<iframe " +
"srcdoc='" + df.head(lines).to_html() + "' " +
"width=1000 height=500>" +
"</iframe>")
return IPython.display.HTML(txt)
Now just run this in any cell:
ldf_display(large_dataframe)
This will convert the dataframe to html then display it in an iframe. The advantage is that you can control the output size and have easily accessible scroll bars.
Worked for my purposes, maybe it will help someone else.