Parsing Multiple Javascript rendered Web pages with Phython and Webkit


import sys
from PyQt4.QtCore import *
from PyQt4.QtGui import *
from PyQt4.QtWebKit import *

class Render(QWebPage):
def __init__(self, urls, cb):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.urls = urls
self.cb = cb
self.crawl()
self.app.exec_()

def crawl(self):
if self.urls:
url = self.urls.pop(0)
print 'Downloading', url
self.mainFrame().load(QUrl(url))
else:
self.app.quit()

def _loadFinished(self, result):
frame = self.mainFrame()
url = str(frame.url().toString())
html = frame.toHtml()
self.cb(url, html)
self.crawl()

def scrape(url, html):
pass # add scraping code here

urls = ['http://webscraping.com', 'http://webscraping.com/blog']
r = Render(urls, cb=scrape)

How to connect ANGULAR with Python via FLASK and CORS

ANGULAR JS



 $http.defaults.headers.post['Content-Type'] = 'application/json';

    $scope.dataa={ 'email' :  'fe','password' :  '12' };

    $http({
        url: 'http://91.134.193.166:5000/',
        method: "POST",
        headers: { 'Content-Type': 'application/json' },
        data: JSON.stringify($scope.dataa)
    })
        .then(function(response) {
            $scope.content = response.data;
                alert(response.data.title);
        },
        function(response) { // optional
             alert( response.data);
        });

PYTHON / FLASK



import json

from flask import Flask, request
from flask_cors import CORS
app = Flask(__name__)
CORS(app)

@app.route('/',methods=['GET', 'POST'])

def write():
    post = request.get_json()
    param = post.get('email')
    with open('/var/www/html/topzapp/Python/somefile.txt', 'w') as the_file:
        the_file.write(param)
    return '{"title": "Person"}'