Root Explorer PHP

[ Root System Explorer ]

Editing: urlExtractorNew.py

# URL extractor
# Copyright 2004, Paul McGuire
from collections import Counter
import pprint
from urllib.request import urlopen

from pyparsing import makeHTMLTags, pyparsing_common as ppc, FollowedBy, trace_parse_action

# Define the pyparsing grammar for a URL, that is:
#    URLlink ::= <a href= URL>linkText</a>
#    URL ::= doubleQuotedString | alphanumericWordPath
# Note that whitespace may appear just about anywhere in the link.  Note also
# that it is not necessary to explicitly show this in the pyparsing grammar; by default,
# pyparsing skips over whitespace between tokens.
linkOpenTag, linkCloseTag = makeHTMLTags("a")
link = linkOpenTag + linkOpenTag.tag_body("body") + linkCloseTag.suppress()

# Add a parse action to expand relative URLs
def expand_relative_url(t):
    url = t.href
    if url.startswith("//"):
        url = "https:" + url
    elif url.startswith(("/", "?", "#")):
        url = "https://www.cnn.com" + url

# Put modified URL back into input tokens
    t["href"] = url

link.add_parse_action(expand_relative_url)

# Go get some HTML with some links in it.
with urlopen("https://www.cnn.com/") as serverListPage:
    htmlText = serverListPage.read().decode()

# scanString is a generator that loops through the input htmlText, and for each
# match yields the tokens and start and end locations (for this application, we are
# not interested in the start and end values).
for toks, strt, end in link.scanString(htmlText):
    print(toks.startA.href, "->", toks.body)

# Create dictionary with a dict comprehension, assembled from each pair of tokens returned
# from a matched URL.
links = {toks.body: toks.href for toks, _, _ in link.scanString(htmlText)}
pprint.pprint(links)

# Parse the urls in the links using pyparsing_common.url, and tally up all
# the different domains in a Counter.
domains = Counter()
for url in links.values():

print(url)
    parsed = ppc.url.parseString(url)

# print parsed fields for each new url
    if parsed.host not in domains:
        print(parsed.dump())
        print()

# update domain counter
    domains[parsed.host] += 1

# Print out a little table of all the domains in the urls
max_domain_len = max(len(d) for d in domains)
print()
print("{:{}s}  {}".format("Domain", max_domain_len, "Count"))
print("{:=<{}}  {:=<5}".format("", max_domain_len, ""))

for domain, count in domains.most_common():
    print("{:{}s}  {:5d}".format(domain, max_domain_len, count))

[ CANCEL ]

Name	Type	Actions
.. (Parent Directory)
📄 0README.html	FILE	[EDIT]
📄 AcManForm.dfm	FILE	[EDIT]
📄 LAparser.py	FILE	[EDIT]
📄 Setup.ini	FILE	[EDIT]
📄 SimpleCalc.py	FILE	[EDIT]
📄 SingleForm.dfm	FILE	[EDIT]
📄 TAP.py	FILE	[EDIT]
📄 __init__.py	FILE	[EDIT]
📄 adventureEngine.py	FILE	[EDIT]
📄 antlr_grammar.py	FILE	[EDIT]
📄 antlr_grammar_tests.py	FILE	[EDIT]
📄 apicheck.py	FILE	[EDIT]
📄 bf.py	FILE	[EDIT]
📄 bigquery_view_parser.py	FILE	[EDIT]
📄 booleansearchparser.py	FILE	[EDIT]
📄 btpyparse.py	FILE	[EDIT]
📄 builtin_parse_action_demo.py	FILE	[EDIT]
📄 cLibHeader.py	FILE	[EDIT]
📄 chemical_formulas.py	FILE	[EDIT]
📄 commasep.py	FILE	[EDIT]
📄 configParse.py	FILE	[EDIT]
📄 cpp_enum_parser.py	FILE	[EDIT]
📄 cuneiform_python.py	FILE	[EDIT]
📄 datetime_parse_actions.py	FILE	[EDIT]
📄 decaf_parser.py	FILE	[EDIT]
📄 delta_time.py	FILE	[EDIT]
📄 dfmparse.py	FILE	[EDIT]
📄 dhcpd_leases_parser.py	FILE	[EDIT]
📄 dictExample.py	FILE	[EDIT]
📄 dictExample2.py	FILE	[EDIT]
📄 directx_x_file_parser.py	FILE	[EDIT]
📄 ebnf.py	FILE	[EDIT]
📄 ebnftest.py	FILE	[EDIT]
📄 email_address_parser.py	FILE	[EDIT]
📄 eval_arith.py	FILE	[EDIT]
📄 excel_expr.py	FILE	[EDIT]
📄 fourFn.py	FILE	[EDIT]
📄 gen_ctypes.py	FILE	[EDIT]
📄 getNTPserversNew.py	FILE	[EDIT]
📄 greeting.py	FILE	[EDIT]
📄 greetingInGreek.py	FILE	[EDIT]
📄 greetingInKorean.py	FILE	[EDIT]
📄 groupUsingListAllMatches.py	FILE	[EDIT]
📄 hola_mundo.py	FILE	[EDIT]
📄 html_stripper.py	FILE	[EDIT]
📄 html_table_parser.py	FILE	[EDIT]
📄 httpServerLogParser.py	FILE	[EDIT]
📄 idlParse.py	FILE	[EDIT]
📄 include_preprocessor.py	FILE	[EDIT]
📄 indentedGrammarExample.py	FILE	[EDIT]
📄 indented_block_example.py	FILE	[EDIT]
📄 inv_regex.py	FILE	[EDIT]
📄 javascript_grammar.g	FILE	[EDIT]
📄 jsonParser.py	FILE	[EDIT]
📄 left_recursion.py	FILE	[EDIT]
📄 lineno_example.py	FILE	[EDIT]
📄 listAllMatches.py	FILE	[EDIT]
📄 lua_parser.py	FILE	[EDIT]
📄 lucene_grammar.py	FILE	[EDIT]
📄 macro_expander.py	FILE	[EDIT]
📄 make_diagram.py	FILE	[EDIT]
📄 matchPreviousDemo.py	FILE	[EDIT]
📄 mongodb_query_expression.py	FILE	[EDIT]
📄 mozilla.ics	FILE	[EDIT]
📄 mozillaCalendarParser.py	FILE	[EDIT]
📄 nested.py	FILE	[EDIT]
📄 nested_markup.py	FILE	[EDIT]
📄 number_words.py	FILE	[EDIT]
📄 numerics.py	FILE	[EDIT]
📄 oc.py	FILE	[EDIT]
📄 one_to_ninety_nine.py	FILE	[EDIT]
📄 parseTabularData.py	FILE	[EDIT]
📄 parse_python_value.py	FILE	[EDIT]
📄 parse_results_sum_example.py	FILE	[EDIT]
📄 partial_gene_match.py	FILE	[EDIT]
📄 pgn.py	FILE	[EDIT]
📄 position.py	FILE	[EDIT]
📄 protobuf_parser.py	FILE	[EDIT]
📄 pythonGrammarParser.py	FILE	[EDIT]
📄 railroad_diagram_demo.py	FILE	[EDIT]
📄 range_check.py	FILE	[EDIT]
📄 readJson.py	FILE	[EDIT]
📄 removeLineBreaks.py	FILE	[EDIT]
📄 roman_numerals.py	FILE	[EDIT]
📄 rosettacode.py	FILE	[EDIT]
📄 scanExamples.py	FILE	[EDIT]
📄 searchParserAppDemo.py	FILE	[EDIT]
📄 searchparser.py	FILE	[EDIT]
📄 select_parser.py	FILE	[EDIT]
📄 sexpParser.py	FILE	[EDIT]
📄 shapes.py	FILE	[EDIT]
📄 simpleArith.py	FILE	[EDIT]
📄 simpleBool.py	FILE	[EDIT]
📄 simpleSQL.py	FILE	[EDIT]
📄 simpleWiki.py	FILE	[EDIT]
📄 snmp_api.h	FILE	[EDIT]
📄 sql2dot.py	FILE	[EDIT]
📄 stackish.py	FILE	[EDIT]
📁 statemachine/	DIR
📄 tag_emitter.py	FILE	[EDIT]
📄 tag_metadata.py	FILE	[EDIT]
📄 test_bibparse.py	FILE	[EDIT]
📄 unicode_denormalizer.py	FILE	[EDIT]
📄 urlExtractor.py	FILE	[EDIT]
📄 urlExtractorNew.py	FILE	[EDIT]
📄 verilog_parse.py	FILE	[EDIT]
📄 withAttribute.py	FILE	[EDIT]
📄 wordsToNum.py	FILE	[EDIT]