MCS 275 Spring 2023 Worksheet 15 Solutions¶

# MCS 275 Worksheet 15 Problem 1
# J Vaccaro
# I completed this work myself, in accordance with the syllabus.

from bs4 import BeautifulSoup
import sys

# Create a beautiful soup from the filename in the first command line arg
with open(sys.argv[1],"r") as infile:
    soup = BeautifulSoup(infile,"html.parser")

# Check whether the soup has a title in the head section, and print a message
if soup.head.title == None:
    print("Warning: No html title. Prettifying anyways!")
else:
    print("Prettifying html with title...",soup.head.title.string)

# Write out the prettified soup to the filename in the 2nd command line arg
with open(sys.argv[2],"wt") as outfile:
    outfile.write(soup.prettify())

from bs4 import BeautifulSoup
from urllib.request import urlopen

with urlopen("https://example.com/") as response:
    soup = BeautifulSoup(response, "html.parser")
    print(soup.prettify())

# It's good practice to save the html locally during development. 
# Here's a short script that saves the html as 'math535.html'

from urllib.request import urlopen
from bs4 import BeautifulSoup

with urlopen("https://www.dumas.io/teaching/2016/spring/math535/") as response:
    soup = BeautifulSoup(response,"html.parser")

with open("math535.html", "wt") as fout:
    fout.write(soup.prettify())

# MCS 275 Worksheet 15 Problem 2
# J Vaccaro
# I completed this work myself, in accordance with the syllabus.

from urllib.request import urlopen
from bs4 import BeautifulSoup

# First, create a beautiful soup, either from the url or a local copy

## --- Comment this out for the final version ------------
# with open("math535.html", "rt") as infile:
#     soup = BeautifulSoup(infile, "html.parser")
## -------------------------------------------------------

# --- Comment this out during development ----------------
with urlopen("https://www.dumas.io/teaching/2016/spring/math535/") as response:
    soup = BeautifulSoup(response,"html.parser")
# --------------------------------------------------------

# We want to make a list of dictionaries, so start with an empty list
hw_data = []

# The relevant section is in an unordered list inside the "homework" div.
hw_ul_tag = soup.find("div",id="homework").ul

# Iterate through each bullet item in the homeworks list
for hw in hw_ul_tag.find_all("li"):

    # Not every 535 homework assignment fits the expected format. 
    # If there's an issue parsing, just skip that assignment and continue.
    # A sweeping try/except is not always recommended, but neither
    # is parsing html.

    try:
        # The problems are inside the contents, on lines without other tags.
        problems = ""

        for prob in hw.contents:
            # Convert to string and strip out starting/ending white space
            prob = str(prob).strip()
            #If the content line has a tag or is whitespace, then skip
            if "<" in prob or prob == "": 
                continue
            #Otherwise, concatenate to problems
            else:
                problems += "\n" + prob

        # The assignment number and title are all inside the "b" tag
        heading = hw.b.string.strip()
        words = heading.split()
        number = int(words[1])
        title = " ".join(words[7:])

        # Create a dictionary with the fields we collected
        d = {"number":number, "title":title, "problems":problems}

        # Append the dictionary to the list of dictionaries
        hw_data.append(d)

    except Exception as e:
        # Skip the homework assignments that don't have the expected format, 
        # but print the error message.
        print(e)
        continue

# Dump out the list-of-dictionaries into a json file.
import json
with open("math535spring2016homework.json", "wt") as outf:
    json.dump(hw_data, outf)

# It's good practice to save the html locally during development. 
# Here's a short script that saves the html as 'capture.html'

from urllib.request import urlopen
from bs4 import BeautifulSoup

with urlopen("https://www.dumas.io/teaching/2021/spring/mcs275/data/capture.html") as response:
    soup = BeautifulSoup(response,"html.parser")

with open("capture.html", "wt") as fout:
    fout.write(soup.prettify())

# MCS 275 Worksheet 15 Problem 3
# J Vaccaro
# I completed this work myself in accordance with the syllabus.

from urllib.request import urlopen
from bs4 import BeautifulSoup

def span_tag_depth(tag):
    """Recursive function for recursing through the span tree and counting the maximum depth.
    Returns the depth."""
    # Maintain a list of the children's maximum depths
    max_span_depth = 0

    # Iterate through the child span tags WITHOUT RECURSING 
    # i.e. only immediate children, not ancestors
    for t in tag.find_all("span", recursive=False):
        depth = span_tag_depth(t)
        # If the child's depth is the deepest so far, then replace.
        if depth>max_span_depth:
            max_span_depth = depth

    # Pass up the maximum depth
    return 1 + max_span_depth

def span_tag_depth_id(tag):
    """Recursive function for recursing through the span tree and counting the maximum depth.
    Returns the depth and the leaf's span id."""
    # Set the default depth
    max_span_depth = 0

    # If the current tag is <span> and has an id, set it as the default id
    # Then, we will "pass up" the leaf id from the longest branch
    if tag.name == "span" and tag.has_attr("id"):
        max_span_id = tag["id"]

    # Iterate through the child span tags WITHOUT RECURSING 
    # i.e. only immediate children, not ancestors
    for t in tag.find_all("span", recursive=False):
        # Recurse through t's children for the max branch length and leaf id
        t_depth,t_id = span_tag_depth_id(t)

        # If t has the deepest depth so far, replace the max depth/id.
        if t_depth>max_span_depth:
            max_span_depth = t_depth
            max_span_id = t_id # leaf id
    
    # Return the augmented max_depth and the id of the leaf.
    return 1+max_span_depth,max_span_id

# Create a beautiful soup, either from the url or a local copy

# --- Comment this out for the final version ------------
with open("capture.html", "rt") as infile:
    soup = BeautifulSoup(infile, "html.parser")
# -------------------------------------------------------

# # --- Comment this out during development ---------------
# with urlopen("https://www.dumas.io/teaching/2021/spring/mcs275/data/capture.html") as response:
#     soup = BeautifulSoup(response,"html.parser")
# # -------------------------------------------------------
print("Maximum depth:",span_tag_depth(soup.span))

depth,span_id = span_tag_depth_id(soup.span)
print("Maximum depth:",depth,"Leaf id:",span_id)

# Here's a bonus solution from Melanie Wertz!
# It uses the indentation properties of prettify() 
# and checks for the line with the most whitespace before a tag.

from bs4 import BeautifulSoup
# I already prettified capture.html to make prettycapture.html
with open("capture.html") as fobj:
    max_depth = 0
    max_id = 0
    for line in fobj:
        if "id" in line:
            opening_whitespace = line.index("<")
            opening_slice = line[0:opening_whitespace]
            if opening_slice.count(" ") > max_depth:
                max_depth = opening_slice.count(" ")
                id_start = line.index("=")
                id_end = line.index(">")
                max_id = line[id_start+1:id_end]
    print(max_id)