In [1]:

import pandas as pd
import numpy as np
import lxml

In [2]:

df = pd.read_html('https://w1.weather.gov/data/obhistory/KDCA.html')

In [7]:

type(df)

Out[7]:

list

In [11]:

df[3]

Out[11]:

	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)				RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure		Precipitation (in.)
	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour		RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in)	sea level(mb)	1 hr	3 hr	6 hr
	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in)	sea level(mb)	1 hr	3 hr	6 hr
0	30	13:52	S 13	10.00	Fair	CLR	66	41	66	47	40%	NaN	NaN	30.31	1026.4	NaN	NaN	NaN
1	30	12:52	S 10	10.00	Fair	CLR	65	40	NaN	NaN	40%	NaN	NaN	30.33	1027.1	NaN	NaN	NaN
2	30	11:52	S 14 G 21	10.00	Fair	CLR	63	38	NaN	NaN	40%	NaN	NaN	30.35	1027.9	NaN	NaN	NaN
3	30	10:52	S 8	10.00	Fair	CLR	58	43	NaN	NaN	58%	NaN	NaN	30.37	1028.5	NaN	NaN	NaN
4	30	09:52	S 8	10.00	Fair	CLR	54	43	NaN	NaN	67%	NaN	NaN	30.38	1028.6	NaN	NaN	NaN
…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…
70	27	15:52	S 10	10.00	Overcast	OVC250	70	45	NaN	NaN	41%	NaN	NaN	30.09	1018.8	NaN	NaN	NaN
71	27	14:52	S 12	10.00	Mostly Cloudy	BKN250	69	49	NaN	NaN	49%	NaN	NaN	30.10	1019.3	NaN	NaN	NaN
72	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
73	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour	6 hour	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
74	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure	Pressure	Precipitation (in.)	Precipitation (in.)	Precipitation (in.)

75 rows × 18 columns

In [ ]:

In [12]:

len(df)

Out[12]:

In [15]:

df[5]

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-15-9258b38c5fd8> in <module>
----> 1 df[5]

IndexError: list index out of range

In [14]:

df[3]

Out[14]:

	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)				RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure		Precipitation (in.)
	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour		RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in)	sea level(mb)	1 hr	3 hr	6 hr
	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in)	sea level(mb)	1 hr	3 hr	6 hr
0	30	13:52	S 13	10.00	Fair	CLR	66	41	66	47	40%	NaN	NaN	30.31	1026.4	NaN	NaN	NaN
1	30	12:52	S 10	10.00	Fair	CLR	65	40	NaN	NaN	40%	NaN	NaN	30.33	1027.1	NaN	NaN	NaN
2	30	11:52	S 14 G 21	10.00	Fair	CLR	63	38	NaN	NaN	40%	NaN	NaN	30.35	1027.9	NaN	NaN	NaN
3	30	10:52	S 8	10.00	Fair	CLR	58	43	NaN	NaN	58%	NaN	NaN	30.37	1028.5	NaN	NaN	NaN
4	30	09:52	S 8	10.00	Fair	CLR	54	43	NaN	NaN	67%	NaN	NaN	30.38	1028.6	NaN	NaN	NaN
…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…
70	27	15:52	S 10	10.00	Overcast	OVC250	70	45	NaN	NaN	41%	NaN	NaN	30.09	1018.8	NaN	NaN	NaN
71	27	14:52	S 12	10.00	Mostly Cloudy	BKN250	69	49	NaN	NaN	49%	NaN	NaN	30.10	1019.3	NaN	NaN	NaN
72	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
73	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour	6 hour	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
74	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure	Pressure	Precipitation (in.)	Precipitation (in.)	Precipitation (in.)

75 rows × 18 columns

In [18]:

df[3].to_pickle('March_30_3day_obs.pickle')

In [22]:

df[3].columns

Out[22]:

MultiIndex([(               'Date',             'Date',             'Date'),
            (          'Time(edt)',        'Time(edt)',        'Time(edt)'),
            (          'Wind(mph)',        'Wind(mph)',        'Wind(mph)'),
            (          'Vis.(mi.)',        'Vis.(mi.)',        'Vis.(mi.)'),
            (            'Weather',          'Weather',          'Weather'),
            (          'Sky Cond.',        'Sky Cond.',        'Sky Cond.'),
            (   'Temperature (ºF)',              'Air',              'Air'),
            (   'Temperature (ºF)',             'Dwpt',             'Dwpt'),
            (   'Temperature (ºF)',           '6 hour',             'Max.'),
            (   'Temperature (ºF)',           '6 hour',             'Min.'),
            (   'RelativeHumidity', 'RelativeHumidity', 'RelativeHumidity'),
            (      'WindChill(°F)',    'WindChill(°F)',    'WindChill(°F)'),
            (      'HeatIndex(°F)',    'HeatIndex(°F)',    'HeatIndex(°F)'),
            (           'Pressure',    'altimeter(in)',    'altimeter(in)'),
            (           'Pressure',    'sea level(mb)',    'sea level(mb)'),
            ('Precipitation (in.)',             '1 hr',             '1 hr'),
            ('Precipitation (in.)',             '3 hr',             '3 hr'),
            ('Precipitation (in.)',             '6 hr',             '6 hr')],
           )

In [23]:

df[3].columns

Out[23]:

MultiIndex([(               'Date',             'Date',             'Date'),
            (          'Time(edt)',        'Time(edt)',        'Time(edt)'),
            (          'Wind(mph)',        'Wind(mph)',        'Wind(mph)'),
            (          'Vis.(mi.)',        'Vis.(mi.)',        'Vis.(mi.)'),
            (            'Weather',          'Weather',          'Weather'),
            (          'Sky Cond.',        'Sky Cond.',        'Sky Cond.'),
            (   'Temperature (ºF)',              'Air',              'Air'),
            (   'Temperature (ºF)',             'Dwpt',             'Dwpt'),
            (   'Temperature (ºF)',           '6 hour',             'Max.'),
            (   'Temperature (ºF)',           '6 hour',             'Min.'),
            (   'RelativeHumidity', 'RelativeHumidity', 'RelativeHumidity'),
            (      'WindChill(°F)',    'WindChill(°F)',    'WindChill(°F)'),
            (      'HeatIndex(°F)',    'HeatIndex(°F)',    'HeatIndex(°F)'),
            (           'Pressure',    'altimeter(in)',    'altimeter(in)'),
            (           'Pressure',    'sea level(mb)',    'sea level(mb)'),
            ('Precipitation (in.)',             '1 hr',             '1 hr'),
            ('Precipitation (in.)',             '3 hr',             '3 hr'),
            ('Precipitation (in.)',             '6 hr',             '6 hr')],
           )

In [28]:

col_list = ['Date','Time(edt)','Wind(mph)','Vis.(mi.)','Weather','Sky','temp_f','temp_dew','temp_6_max','temp_6_min','RelativeHumidity','WindChill(°F)','HeatIndex(°F)','Pressure_in','Pressure_mb','precip_1','precip_2','precip_3']

In [29]:

len(col_list)

Out[29]:

In [30]:

len(df[3].columns)

Out[30]:

In [31]:

obs1 = df[3].copy()

In [32]:

obs1

Out[32]:

	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)				RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure		Precipitation (in.)
	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour		RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in)	sea level(mb)	1 hr	3 hr	6 hr
	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in)	sea level(mb)	1 hr	3 hr	6 hr
0	30	13:52	S 13	10.00	Fair	CLR	66	41	66	47	40%	NaN	NaN	30.31	1026.4	NaN	NaN	NaN
1	30	12:52	S 10	10.00	Fair	CLR	65	40	NaN	NaN	40%	NaN	NaN	30.33	1027.1	NaN	NaN	NaN
2	30	11:52	S 14 G 21	10.00	Fair	CLR	63	38	NaN	NaN	40%	NaN	NaN	30.35	1027.9	NaN	NaN	NaN
3	30	10:52	S 8	10.00	Fair	CLR	58	43	NaN	NaN	58%	NaN	NaN	30.37	1028.5	NaN	NaN	NaN
4	30	09:52	S 8	10.00	Fair	CLR	54	43	NaN	NaN	67%	NaN	NaN	30.38	1028.6	NaN	NaN	NaN
…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…
70	27	15:52	S 10	10.00	Overcast	OVC250	70	45	NaN	NaN	41%	NaN	NaN	30.09	1018.8	NaN	NaN	NaN
71	27	14:52	S 12	10.00	Mostly Cloudy	BKN250	69	49	NaN	NaN	49%	NaN	NaN	30.10	1019.3	NaN	NaN	NaN
72	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
73	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour	6 hour	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
74	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure	Pressure	Precipitation (in.)	Precipitation (in.)	Precipitation (in.)

75 rows × 18 columns

In [33]:

obs1.columns = col_list

In [34]:

obs1

Out[34]:

	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky	temp_f	temp_dew	temp_6_max	temp_6_min	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure_in	Pressure_mb	precip_1	precip_2	precip_3
0	30	13:52	S 13	10.00	Fair	CLR	66	41	66	47	40%	NaN	NaN	30.31	1026.4	NaN	NaN	NaN
1	30	12:52	S 10	10.00	Fair	CLR	65	40	NaN	NaN	40%	NaN	NaN	30.33	1027.1	NaN	NaN	NaN
2	30	11:52	S 14 G 21	10.00	Fair	CLR	63	38	NaN	NaN	40%	NaN	NaN	30.35	1027.9	NaN	NaN	NaN
3	30	10:52	S 8	10.00	Fair	CLR	58	43	NaN	NaN	58%	NaN	NaN	30.37	1028.5	NaN	NaN	NaN
4	30	09:52	S 8	10.00	Fair	CLR	54	43	NaN	NaN	67%	NaN	NaN	30.38	1028.6	NaN	NaN	NaN
…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…	…
70	27	15:52	S 10	10.00	Overcast	OVC250	70	45	NaN	NaN	41%	NaN	NaN	30.09	1018.8	NaN	NaN	NaN
71	27	14:52	S 12	10.00	Mostly Cloudy	BKN250	69	49	NaN	NaN	49%	NaN	NaN	30.10	1019.3	NaN	NaN	NaN
72	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
73	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour	6 hour	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
74	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure	Pressure	Precipitation (in.)	Precipitation (in.)	Precipitation (in.)

75 rows × 18 columns

In [37]:

obs1.tail(3)

Out[37]:

	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky	temp_f	temp_dew	temp_6_max	temp_6_min	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure_in	Pressure_mb	precip_1	precip_2	precip_3
72	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	Max.	Min.	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
73	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Air	Dwpt	6 hour	6 hour	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	altimeter(in.)	sea level(mb)	1 hr	3 hr	6 hr
74	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky Cond.	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	Temperature (ºF)	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure	Pressure	Precipitation (in.)	Precipitation (in.)	Precipitation (in.)

In [38]:

obs1.drop(obs1.tail(3).index,inplace=True)

In [39]:

obs1.tail()

Out[39]:

	Date	Time(edt)	Wind(mph)	Vis.(mi.)	Weather	Sky	temp_f	temp_dew	temp_6_max	temp_6_min	RelativeHumidity	WindChill(°F)	HeatIndex(°F)	Pressure_in	Pressure_mb	precip_1	precip_2	precip_3
67	27	18:52	S 9 G 22	10.00	Overcast	SCT120 BKN150 OVC250	66	47	NaN	NaN	50%	NaN	NaN	30.11	1019.5	NaN	NaN	NaN
68	27	17:52	S 8	10.00	Overcast	FEW120 BKN150 OVC250	67	49	NaN	NaN	53%	NaN	NaN	30.09	1018.7	NaN	NaN	NaN
69	27	16:52	S 10	10.00	Overcast	BKN160 OVC250	68	49	NaN	NaN	51%	NaN	NaN	30.08	1018.4	NaN	NaN	NaN
70	27	15:52	S 10	10.00	Overcast	OVC250	70	45	NaN	NaN	41%	NaN	NaN	30.09	1018.8	NaN	NaN	NaN
71	27	14:52	S 12	10.00	Mostly Cloudy	BKN250	69	49	NaN	NaN	49%	NaN	NaN	30.10	1019.3	NaN	NaN	NaN

In [42]:

obs1.to_csv('March_27_30_data.csv')

loading 3 day weather data using pandas

Notes

Leave a ReplyCancel reply