Paweł Domański
119 lines
4.1 KiB
Python
119 lines
4.1 KiB
Python
import time
|
|
import pandas as pd
|
|
import undetected_chromedriver as uc
|
|
from bs4 import BeautifulSoup
|
|
from selenium.webdriver.common.keys import Keys
|
|
import os
|
|
import json
|
|
|
|
def scrape_alternativeto(i):
|
|
url = f'https://alternativeto.net/category/business-and-commerce/cms/?p={i}'
|
|
|
|
# Konfiguracja Selenium (undetected-chromedriver)
|
|
options = uc.ChromeOptions()
|
|
options.add_argument("--disable-blink-features=AutomationControlled")
|
|
|
|
driver = uc.Chrome(options=options)
|
|
driver.get(url)
|
|
|
|
# Poczekaj na załadowanie strony
|
|
time.sleep(2)
|
|
|
|
# Akceptacja cookies jeśli pojawi się banner
|
|
try:
|
|
consent_button = driver.find_element("xpath", "//button[contains(., 'Consent') or contains(., 'Zgadzam się')]")
|
|
consent_button.click()
|
|
time.sleep(1)
|
|
except Exception:
|
|
pass # Jeśli nie ma bannera, przechodzimy dalej
|
|
|
|
# Przewiń stronę, aby załadować więcej CMS-ów
|
|
body = driver.find_element("tag name", "body")
|
|
for _ in range(10): # Możesz zwiększyć, jeśli nie pobiera wszystkich wyników
|
|
body.send_keys(Keys.PAGE_DOWN)
|
|
time.sleep(1)
|
|
|
|
# Pobierz HTML strony
|
|
html = driver.page_source
|
|
driver.quit()
|
|
|
|
# Parsowanie HTML za pomocą BeautifulSoup
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
# **🔍 ZMIENNE: Ustaw poprawne selektory CSS dla CMS-ów**
|
|
cms_items = soup.select("div[id^='app-item-']") # Pobiera wszystkie CMS-y
|
|
|
|
if not cms_items:
|
|
print("⚠️ Nie znaleziono CMS-ów! Sprawdź poprawny selektor w przeglądarce.")
|
|
return
|
|
|
|
# Lista na wyniki
|
|
extracted_data = []
|
|
|
|
for item in cms_items:
|
|
# Nazwa narzędzia
|
|
name_tag = item.select_one("div.flex.flex-col.w-full.gap-3 > div > div.min-w-\\[180px\\].md\\:min-w-0 > a > h2")
|
|
name = name_tag.text.strip() if name_tag else "N/A"
|
|
|
|
# Link
|
|
link_tag = item.select_one("div.flex.flex-col.w-full.gap-3 > div > div.min-w-\\[180px\\].md\\:min-w-0 > a")
|
|
link = f"https://alternativeto.net{link_tag['href']}" if link_tag and link_tag.has_attr('href') else "N/A"
|
|
|
|
# Obrazek
|
|
image_tag = item.select_one("div.flex.flex-col.w-full.gap-3 > div > div.flex.items-center.lg\\:hidden > a > div > img")
|
|
image = image_tag['src'] if image_tag and image_tag.has_attr('src') else "N/A"
|
|
|
|
extracted_data.append({
|
|
"name": name,
|
|
"link": link,
|
|
"image": image
|
|
})
|
|
|
|
if not extracted_data:
|
|
print("⚠️ Nie udało się wydobyć żadnych danych.")
|
|
return
|
|
|
|
# Dopisywanie do istniejących danych JSON (jeśli plik istnieje)
|
|
json_path = "cms_alternatives.json"
|
|
if os.path.exists(json_path):
|
|
with open(json_path, "r", encoding="utf-8") as f:
|
|
try:
|
|
existing_data = json.load(f)
|
|
except Exception:
|
|
existing_data = []
|
|
else:
|
|
existing_data = []
|
|
|
|
# Dodaj nowe dane, unikając duplikatów po nazwie i linku
|
|
existing_keys = {(d["name"], d["link"]) for d in existing_data if "name" in d and "link" in d}
|
|
for row in extracted_data:
|
|
if (row["name"], row["link"]) not in existing_keys:
|
|
existing_data.append(row)
|
|
|
|
# Zapisz zaktualizowane dane
|
|
with open(json_path, "w", encoding="utf-8") as f:
|
|
json.dump(existing_data, f, indent=4, ensure_ascii=False)
|
|
print("✅ Dane dopisane do cms_alternatives.json")
|
|
|
|
# Analogicznie dla CSV
|
|
import pandas as pd
|
|
df_new = pd.DataFrame(extracted_data)
|
|
csv_path = "cms_alternatives.csv"
|
|
if os.path.exists(csv_path):
|
|
df_existing = pd.read_csv(csv_path, encoding="utf-8")
|
|
df_all = pd.concat([df_existing, df_new], ignore_index=True)
|
|
df_all = df_all.drop_duplicates(subset=["name", "link"])
|
|
else:
|
|
df_all = df_new
|
|
df_all.to_csv(csv_path, index=False, encoding="utf-8")
|
|
print("✅ Dane dopisane do cms_alternatives.csv")
|
|
|
|
# Uruchamiamy skrypt
|
|
if __name__ == "__main__":
|
|
for i in range(15):
|
|
print(f"🔄 Strona {i + 12}...")
|
|
j = i + 12
|
|
scrape_alternativeto(j)
|
|
time.sleep(2)
|
|
## scrape_alternativeto()
|