Web scraping
In [1]:
import requests
In [6]:
r = requests.get('http://blue.math.buffalo.edu')
In [3]:
r
Out[3]:
In [4]:
r.text
Out[4]:
In [5]:
ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299"
In [7]:
user_agent = {'User-agent': ua}
r = requests.get('http://blue.math.buffalo.edu',headers=user_agent)
In [8]:
import pandas
In [9]:
df = pandas.read_csv('airports.csv')
In [10]:
df.head()
Out[10]:
In [12]:
def foo(x):
return x
myconverters = {'continent':foo}
df = pandas.read_csv('airports.csv',converters=myconverters)
df.head(2)
Out[12]:
In [13]:
df['type']
Out[13]:
In [14]:
df[['latitude_deg','longitude_deg']]
Out[14]:
In [15]:
len(df)
Out[15]:
In [16]:
import matplotlib.pyplot as plt
%matplotlib inline
In [25]:
plt.figure(figsize=(12,8))
plt.plot(df['longitude_deg'],df['latitude_deg'],'bo',alpha=0.05,ms=2)
plt.savefig('airports.png')
In [26]:
df['latitude_deg'].max()
Out[26]:
In [29]:
imax = df['latitude_deg'].argmax()
imax
Out[29]:
In [30]:
df.iloc[imax]
Out[30]:
In [31]:
df['type'].value_counts()
Out[31]:
In [33]:
df[['latitude_deg','longitude_deg']].to_csv('temp.csv',index=False,sep='\t')
In [34]:
df[['latitude_deg','longitude_deg']].to_excel('temp.xlsx')
In [35]:
?pandas.read_csv