How do I iterate over different links to create a dataframe in pandas?

I have been coding a script to scrape the premier league website for players. It will go into each player page from the main page then scrape the information specified from the table but I cannot loop it yet. I understand it probably is super verbose and horrible but I am still learning. I have stored a list of 843 links which I want to iterate through on both /overview and /stats links. How do I go about this? Thanks.

# Code  player_list = pl_links  player = 'https://www.premierleague.com/players/13549/Tosin-Adarabioyo'  urlmod1 = player+'/overview'  urlmod2 = player+'/stats'    # Change url to overview page **************************************************************************************************  def df_function():      url = urlmod1      html = urlopen(url)      soup = BeautifulSoup(html, 'lxml')      type(soup)        # Create column names as a list       col_items = []      row_items = []        # Use dictionary to pass key : value pair      cols = soup.find_all('div', {"class" : "label"})[:6]      rows = soup.find_all('div', {"class" : "info"})[:6]        for col in cols:      #    print (col.get_text())          col_items.append((col.get_text()))      for row in rows:      #    print(row.get_text())          row_items.append((row.get_text()))        # Clean rows list before appending      row_items_clean = []        for elem in row_items:              row_items_clean.extend(elem.strip().split('\n'))        df = pd.DataFrame([row_items_clean], columns=col_items)      df        # Change url to stat page ******************************************************************************************************      url = urlmod2      html = urlopen(url)      soup = BeautifulSoup(html, 'lxml')      type(soup)        # Clean output      sep = '\n'      stripped = text.split(sep, 1)[0]        # Create column names as a list      col_stats = []      row_stats = []        # Use dictionary to pass key : value pair      c_stats = soup.find_all('span', {"class" : "stat"})      r_stats = soup.find_all('span', {"class" : "allStatContainer"})        for col in c_stats:      #    print (col.get_text())          col_stats.append((col.get_text().split(sep, 1)[0]))      for row in r_stats:      #    print(row.get_text())          row_stats.append((row.get_text()))        # Remove unused table titles      col_stats      for elem in list(col_stats):          if elem == 'Attack' or elem == 'Team Play' or elem == 'Discipline' or elem == 'Defence':              col_stats.remove(elem)        # Clean columns list before appending      col_stats_clean = []        for i in col_stats:          j = i.replace('   ', '')          col_stats_clean.append(j)      #print(col_stats_clean)        # Clean rows list before appending      row_stats_clean = []        for elem in row_stats:              row_stats_clean.extend(elem.strip().split('\n'))        #print(row_stats_clean)        # Remove unused table titles      col_stats      for elem in list(col_stats):          if elem == 'Attack' or elem == 'Team Play' or elem == 'Discipline' or elem == 'Defence':              col_stats.remove(elem)        # Clean columns list before appending      col_stats_clean = []        for i in col_stats:          j = i.replace('   ', '')          col_stats_clean.append(j)      #print(col_stats_clean)        # Clean rows list before appending      row_stats_clean = []        for elem in row_stats:              row_stats_clean.extend(elem.strip().split('\n'))        #print(row_stats_clean)        # Creating dataframe for stats *************************************************************************************************      stats_df = pd.DataFrame([row_stats_clean], columns=col_stats_clean)      #stats_df        join_1 = df.join(stats_df, how='right')        # Create names as a list      row_names = []        # Use dictionary to pass key : value pair      r_names = soup.find_all('div', {"class" : "name t-colour"})        for row in r_names:          row_names.append((row.get_text()))      #print(row.get_text())        # Create dataframe for names      names_df = pd.DataFrame([row_names])      names_df.columns = ['Name']        pl_player_stats = names_df.join(join_1, how='right')        # Result      return pl_player_stats  

It should return a df like this:

Name    Club    Position    Nationality Date of Birth   Height  Weight  Appearances Goals   Wins    Losses  Clean sheets    Goals Conceded  Tackles Tackle success %    Last man tackles    Blocked shots   Interceptions   Clearances  Headed Clearance    Clearances off line Recoveries  Duels won   Duels lost  Successful 50/50s   Aerial battles won  Aerial battles lost Own goals   Errors leading to goal  Assists Passes  Passes per match    Big Chances Created Crosses Cross accuracy %    Through balls   Accurate long balls Yellow cards    Red cards   Fouls   Offsides    Goals   Headed goals    Goals with right foot   Goals with left foot    Hit woodwork  0   Tosin Adarabioyo    Fulham  Defender    England 24/09/1997 (23) 196cm   80kg    26  0   5   10  9   27  29  62% 0   1   35  154 78  0   115 110 64  10  68  31  1   0   0   1,466   56.38   0   2   2%  0   99  1   0   13  1   0   0   0   0   0  

https://stackoverflow.com/questions/66727139/how-do-i-iterate-over-different-links-to-create-a-dataframe-in-pandas March 21, 2021 at 06:36AM