Hi All! I’m looking to build a personal scraper for MLS listings that I can automate into a spreadsheet and also look into if the address meets opportunity zones and get an idea of the property’s avg income data based on 'https://geomap.ffiec.gov/ffiecgeomap/. I am currently in step one. How would I go about doing this? I receive an error ‘"aborted.’, RemoteDisconnected(‘Remote end closed connection without response’)
Repl link:
import requests
from bs4 import BeautifulSoup
# URL of MLS listings page
url = "mlspin_url_here"
print(url)
# Credentials for logging in
username = "my_username_here"
password = "my_password_here"
def login_session(username, password):
# Create a session object to persist cookies across requests
session = requests.Session()
# URL of the login page
login_url = "mls_login_url_here"
# Parameters for the login POST request
login_data = {
'ctl00$Body$Login$UserName': username,
'ctl00$Body$Login$Password': password,
'ctl00$Body$Login$LoginButton': 'Sign In'
}
# Send a POST request to login
response = session.post(login_url, data=login_data)
# Check if login was successful (redirects to another page)
if response.status_code == 200 and "Redirect" in response.text:
print("Login successful.")
return session
else:
print("Login failed.")
return None
def scrape_mls_listings(url):
try:
# Send a GET request to the URL
response = requests.get(url)
# Check if the request was successful (status code 200)
if response.status_code == 200:
# Parse the HTML content
soup = BeautifulSoup(response.content, 'html.parser')
# Find all listing items
listings = soup.find_all('div', class_='listing')
# Iterate over each listing and extract relevant information
for listing in listings:
# Extract details like address, price, etc.
address = listing.find('div', class_='address').text.strip()
price = listing.find('div', class_='price').text.strip()
beds = listing.find('div', class_='beds').text.strip()
baths = listing.find('div', class_='baths').text.strip()
sqft = listing.find('div', class_='sqft').text.strip()
# Print the extracted information
print(f"Address: {address}")
print(f"Price: {price}")
print(f"Beds: {beds}")
print(f"Baths: {baths}")
print(f"Square Feet: {sqft}")
print("-----------------------------------------")
else:
print("Failed to retrieve MLS listings. Status code:", response.status_code)
except requests.exceptions.RequestException as e:
print("An error occurred during the request:", e)
# Call the function to scrape MLS listings
scrape_mls_listings(url)