{

"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import quandl\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"[
0
1
2
3\n",
" 0 Abbreviation
State Name
Capital
Became a State\n",
" 1
AL
Alabama
Montgomery December 14, 1819\n",
" 2
AK
Alaska
Juneau
January 3, 1959\n",
" 3
AZ
Arizona
Phoenix February 14, 1912\n",
" 4
AR
Arkansas
Little Rock
June 15, 1836\n",
" 5
CA
California
Sacramento September 9, 1850\n",
" 6
CO
Colorado
Denver
August 1, 1876\n",
" 7
CT
Connecticut
Hartford
January 9, 1788\n",
" 8
DE
Delaware
Dover December 7, 1787\n",
" 9
FL
Florida
Tallahassee
March 3, 1845\n",
" 10
GA
Georgia
Atlanta
January 2, 1788\n",
" 11
HI
Hawaii
Honolulu
August 21, 1959\n",
" 12
ID
Idaho
Boise
July 3, 1890\n",
" 13
IL
Illinois
Springfield December 3, 1818\n",
" 14
IN
Indiana
Indianapolis December 11, 1816\n",
" 15
IA
Iowa
Des Moines December 28, 1846\n",
" 16
KS
Kansas
Topeka January 29, 1861\n",
" 17
KY
Kentucky
Frankfort
June 1, 1792\n",
" 18
LA
Louisiana
Baton Rouge
April 30, 1812\n",
" 19
ME
Maine
Augusta
March 15, 1820\n",
" 20
MD
Maryland
Annapolis
April 28, 1788\n",
" 21
MA Massachusetts
Boston February 6, 1788\n",
" 22
MI
Michigan
Lansing January 26, 1837\n",
" 23
MN
Minnesota
Saint Paul
May 11, 1858\n",
" 24
MS
Mississippi
Jackson December 10, 1817\n",
" 25
MO
Missouri Jefferson City
August 10, 1821\n",
" 26
MT
Montana
Helena November 8, 1889\n",
" 27
NE
Nebraska
Lincoln
March 1, 1867\n",
" 28
NV
Nevada
Carson City October 31, 1864\n",
" 29
NH New Hampshire
Concord
June 21, 1788\n",
" 30
NJ
New Jersey
Trenton December 18, 1787\n",
" 31
NM
New Mexico
Santa Fe
January 6, 1912\n",
" 32
NY
New York
Albany
July 26, 1788\n",
" 33
NC North Carolina
Raleigh November 21, 1789\n",
" 34
ND
North Dakota
Bismarck November 2, 1889\n",

"
"
"
"
"
"
"
"
"
"
"
"
"
"
"
"

35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

OH
Ohio
Columbus
March 1, 1803\n",
OK
Oklahoma Oklahoma City November 16, 1907\n",
OR
Oregon
Salem February 14, 1859\n",
PA
Pennsylvania
Harrisburg December 12, 1787\n",
RI
Rhode Island
Providence
May 19, 1790\n",
SC South Carolina
Columbia
May 23, 1788\n",
SD
South Dakota
Pierre November 2, 1889\n",
TN
Tennessee
Nashville
June 1, 1796\n",
TX
Texas
Austin December 29, 1845\n",
UT
Utah Salt Lake City
January 4, 1896\n",
VT
Vermont
Montpelier
March 4, 1791\n",
VA
Virginia
Richmond
June 25, 1788\n",
WA
Washington
Olympia November 11, 1889\n",
WV West Virginia
Charleston
June 20, 1863\n",
WI
Wisconsin
Madison
May 29, 1848\n",
WY
Wyoming
Cheyenne
July 10, 1890,\n"

"
"
"
"
"
"
"
"
"
"
"
"
"

0 \\\n",
0 v t e Political divisions of the United States... \n",
1
v t e Political divisions of the United States \n",
2
NaN \n",
3
States \n",
4
NaN \n",
5
National capital \n",
6
NaN \n",
7
Large islands \n",
8
NaN \n",
9
Small islands \n",
\n",
1 2
3

\\\n"

6
NaN
NaN
NaN
NaN
NaN
NaN
NaN

" 0
v t e Political divisions of the United States
" 1
NaN
" 2
NaN
" 3 Alabama Alaska Arizona Arkansas Califo...
" 4
NaN
" 5
District of Columbia
" 6
NaN
" 7 AmericanSamoa Guam NorthernMarianaIslan...
" 8
NaN
" 9 BakerIsland HowlandIsland JarvisIsland...
" \n",
"
4
\\\n",
" 0 Alabama Alaska Arizona Arkansas Califo...
" 1
NaN
\n",
" 2
NaN
\n",
" 3
NaN
\n",
" 4
NaN
\n",
" 5
NaN
\n",
" 6
NaN
\n",
" 7
NaN
\n",
" 8
NaN

NaN States
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN

\n",
\n",
\n",
\n",
\n",
\n",
\n",
\n",
\n",
\n",

5
NaN National capital
NaN
NaN
NaN
NaN
NaN
NaN
NaN
NaN

\n",

NaN

\n",
" 9
NaN NaN
NaN \n",
" \n",
"
7 8
9 \\\n",
" 0 District of Columbia NaN Large islands \n",
" 1
NaN NaN
NaN \n",
" 2
NaN NaN
NaN \n",
" 3
NaN NaN
NaN \n",
" 4
NaN NaN
NaN \n",
" 5
NaN NaN
NaN \n",
" 6
NaN NaN
NaN \n",
" 7
NaN NaN
NaN \n",
" 8
NaN NaN
NaN \n",
" 9
NaN NaN
NaN \n",
" \n",
"
10 11
12
\\\n",
" 0 AmericanSamoa Guam NorthernMarianaIslan... NaN Small islands
" 1
NaN NaN
NaN
\n",
" 2
NaN NaN
NaN
\n",
" 3
NaN NaN
NaN
\n",
" 4
NaN NaN
NaN
\n",
" 5
NaN NaN
NaN
\n",
" 6
NaN NaN
NaN
\n",
" 7
NaN NaN
NaN
\n",
" 8
NaN NaN
NaN
\n",
" 9
NaN NaN
NaN
\n",
" \n",
"
13 \n",
" 0 BakerIsland HowlandIsland JarvisIsland... \n",
" 1
NaN \n",
" 2
NaN \n",
" 3
NaN \n",
" 4
NaN \n",
" 5
NaN \n",
" 6
NaN \n",
" 7
NaN \n",
" 8
NaN \n",
" 9
NaN ,\n",
"
0 \\\n",
" 0 v t e Political divisions of the United States \n",
" 1
NaN \n",
" 2
States \n",
" 3
NaN \n",
" 4
National capital \n",
" 5
NaN \n",
" 6
Large islands \n",
" 7
NaN \n",
" 8
Small islands \n",
" \n",

\n",

"
"
"
"
"
"
"
"
"
"

1
0
NaN
1
NaN
2 Alabama Alaska Arizona Arkansas Califo...
3
NaN
4
District of Columbia
5
NaN
6 AmericanSamoa Guam NorthernMarianaIslan...
7
NaN
8 BakerIsland HowlandIsland JarvisIsland...

\n",
\n",
\n",
\n",
\n",
\n",
\n",
\n",
\n",
]"

]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"fiddy_states = pd.read_html('https://simple.wikipedia.org/wiki/List_of_U.S.
_states')\n",
"fiddy_states"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"mainHPIlist = pd.DataFrame()\n",
"\n",
"for abbv in fiddy_states[0][0][1:3]:\n",
"
query = \"FMAC/HPI_\"+str(abbv)\n",
"
df = quandl.get(query)\n",
"
df.reset_index(inplace=True)\n",
"
\n",
"
if mainHPIlist.empty:\n",
"
mainHPIlist = df\n",
"
else:\n",
"
mainHPIlist = pd.merge(mainHPIlist, df, on='Date')"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
"
<tr style=\"text-align: right;\">\n",
"
<th></th>\n",
"
<th>Date</th>\n",

"
<th>Value_x</th>\n",
"
<th>Value_y</th>\n",
"
</tr>\n",
" </thead>\n",
" <tbody>\n",
"
<tr>\n",
"
<th>0</th>\n",
"
<td>1975-01-31</td>\n",
"
<td>35.352518</td>\n",
"
<td>34.429922</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>1</th>\n",
"
<td>1975-02-28</td>\n",
"
<td>35.560502</td>\n",
"
<td>34.940483</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>2</th>\n",
"
<td>1975-03-31</td>\n",
"
<td>35.808615</td>\n",
"
<td>35.463028</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>3</th>\n",
"
<td>1975-04-30</td>\n",
"
<td>36.118785</td>\n",
"
<td>36.008230</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>4</th>\n",
"
<td>1975-05-31</td>\n",
"
<td>36.418630</td>\n",
"
<td>36.603608</td>\n",
"
</tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"
Date
Value_x
Value_y\n",
"0 1975-01-31 35.352518 34.429922\n",
"1 1975-02-28 35.560502 34.940483\n",
"2 1975-03-31 35.808615 35.463028\n",
"3 1975-04-30 36.118785 36.008230\n",
"4 1975-05-31 36.418630 36.603608"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mainHPIlist.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},

"source": [
"### Pickling\n",
"This is the method of saving Python objects as byte files."
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import pickle"
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"pickle_out = open('fiddy_states.pickle', 'wb')\n",
"pickle.dump(mainHPIlist, pickle_out)\n",
"pickle_out.close()"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"collapsed": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"
Date
Value_x
Value_y\n",
"0 1975-01-31 35.352518 34.429922\n",
"1 1975-02-28 35.560502 34.940483\n",
"2 1975-03-31 35.808615 35.463028\n",
"3 1975-04-30 36.118785 36.008230\n",
"4 1975-05-31 36.418630 36.603608\n",
"5 1975-06-30 36.590431 37.240909\n",
"6 1975-07-31 36.565213 37.878196\n",
"7 1975-08-31 36.374700 38.458854\n",
"8 1975-09-30 36.138194 38.964821\n",
"9 1975-10-31 36.012989 39.425012\n",
"10 1975-11-30 36.092945 39.835432\n",
"11 1975-12-31 36.406437 40.176140\n",
"12 1976-01-31 36.929302 40.449443\n",
"13 1976-02-29 37.538576 40.678293\n",
"14 1976-03-31 38.078041 40.890297\n",
"15 1976-04-30 38.453933 41.098638\n",
"16 1976-05-31 38.668723 41.305256\n",
"17 1976-06-30 38.766728 41.505532\n",
"18 1976-07-31 38.807778 41.676104\n",

"19 1976-08-31 38.804585

"20 1976-09-30 38.778522
"21 1976-10-31 38.810817
"22 1976-11-30 38.958589
"23 1976-12-31 39.132770
"24 1977-01-31 39.227060
"25 1977-02-28 39.268027
"26 1977-03-31 39.329486
"27 1977-04-30 39.518284
"28 1977-05-31 39.842819
"29 1977-06-30 40.260273
"..
...
...
"468 2014-01-31 119.807575
"469 2014-02-28 120.616671
"470 2014-03-31 121.368329
"471 2014-04-30 122.106052
"472 2014-05-31 122.916856
"473 2014-06-30 123.799021
"474 2014-07-31 124.525359
"475 2014-08-31 124.755697
"476 2014-09-30 124.292233
"477 2014-10-31 123.481198
"478 2014-11-30 122.918330
"479 2014-12-31 122.800455
"480 2015-01-31 123.065741
"481 2015-02-28 123.533474
"482 2015-03-31 124.296094
"483 2015-04-30 125.317995
"484 2015-05-31 126.417577
"485 2015-06-30 127.332480
"486 2015-07-31 127.755998
"487 2015-08-31 127.652336
"488 2015-09-30 127.138274
"489 2015-10-31 126.563561
"490 2015-11-30 126.109320
"491 2015-12-31 125.859100
"492 2016-01-31 126.136235
"493 2016-02-29 127.006956
"494 2016-03-31 128.217720
"495 2016-04-30 129.578002
"496 2016-05-31 130.687757
"497 2016-06-30 131.888538
"\n",
"[498 rows x 3 columns]\n"

41.807870\n",
41.966723\n",
42.212437\n",
42.541828\n",
42.938783\n",
43.363485\n",
43.758560\n",
44.077088\n",
44.319448\n",
44.528720\n",
44.736402\n",
...\n",
163.947045\n",
164.757702\n",
166.116452\n",
167.519706\n",
168.734417\n",
169.735794\n",
170.392760\n",
170.569661\n",
170.222200\n",
169.483581\n",
168.641965\n",
168.092263\n",
168.025068\n",
168.487183\n",
169.576576\n",
171.237447\n",
173.168144\n",
174.879180\n",
175.969707\n",
176.450886\n",
176.412634\n",
176.002921\n",
175.604807\n",
175.219557\n",
174.785824\n",
174.591956\n",
174.919627\n",
175.844104\n",
177.140103\n",
178.187920\n",

]
}
],
"source": [
"pickle_in = open('fiddy_states.pickle','rb')\n",
"HPI_data = pickle.load(pickle_in)\n",
"print(HPI_data)"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {
"collapsed": false
},
"outputs": [

{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
"
<tr style=\"text-align: right;\">\n",
"
<th></th>\n",
"
<th>Date</th>\n",
"
<th>Value_x</th>\n",
"
<th>Value_y</th>\n",
"
</tr>\n",
" </thead>\n",
" <tbody>\n",
"
<tr>\n",
"
<th>0</th>\n",
"
<td>1975-01-31</td>\n",
"
<td>35.352518</td>\n",
"
<td>34.429922</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>1</th>\n",
"
<td>1975-02-28</td>\n",
"
<td>35.560502</td>\n",
"
<td>34.940483</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>2</th>\n",
"
<td>1975-03-31</td>\n",
"
<td>35.808615</td>\n",
"
<td>35.463028</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>3</th>\n",
"
<td>1975-04-30</td>\n",
"
<td>36.118785</td>\n",
"
<td>36.008230</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>4</th>\n",
"
<td>1975-05-31</td>\n",
"
<td>36.418630</td>\n",
"
<td>36.603608</td>\n",
"
</tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"
Date
Value_x
Value_y\n",
"0 1975-01-31 35.352518 34.429922\n",
"1 1975-02-28 35.560502 34.940483\n",
"2 1975-03-31 35.808615 35.463028\n",
"3 1975-04-30 36.118785 36.008230\n",
"4 1975-05-31 36.418630 36.603608"
]
},
"execution_count": 36,
"metadata": {},
"output_type": "execute_result"

}
],
"source": [
"HPI_data.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Python has its own pickle module"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
"
<tr style=\"text-align: right;\">\n",
"
<th></th>\n",
"
<th>Date</th>\n",
"
<th>Value_x</th>\n",
"
<th>Value_y</th>\n",
"
</tr>\n",
" </thead>\n",
" <tbody>\n",
"
<tr>\n",
"
<th>0</th>\n",
"
<td>1975-01-31</td>\n",
"
<td>35.352518</td>\n",
"
<td>34.429922</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>1</th>\n",
"
<td>1975-02-28</td>\n",
"
<td>35.560502</td>\n",
"
<td>34.940483</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>2</th>\n",
"
<td>1975-03-31</td>\n",
"
<td>35.808615</td>\n",
"
<td>35.463028</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>3</th>\n",
"
<td>1975-04-30</td>\n",
"
<td>36.118785</td>\n",
"
<td>36.008230</td>\n",
"
</tr>\n",
"
<tr>\n",
"
<th>4</th>\n",

"
<td>1975-05-31</td>\n",
"
<td>36.418630</td>\n",
"
<td>36.603608</td>\n",
"
</tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"
Date
Value_x
Value_y\n",
"0 1975-01-31 35.352518 34.429922\n",
"1 1975-02-28 35.560502 34.940483\n",
"2 1975-03-31 35.808615 35.463028\n",
"3 1975-04-30 36.118785 36.008230\n",
"4 1975-05-31 36.418630 36.603608"
]
},
"execution_count": 37,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mainHPIlist.to_pickle('pickle.pickle')\n",
"HPI_data2 = pd.read_pickle('pickle.pickle')\n",
"HPI_data2.head()"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "Python [Root]",
"language": "python",
"name": "Python [Root]"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 0
}