Multi-threaded parsing/downloading
This commit is contained in:
parent
aa9755b108
commit
0e0bb2ed91
1
Makefile
1
Makefile
|
@ -35,6 +35,7 @@ SRCS = \
|
|||
src/Entry.cpp, \
|
||||
src/EntriesView.cpp, \
|
||||
src/Feed.cpp, \
|
||||
src/FeedController.cpp \
|
||||
src/FeedsView.cpp, \
|
||||
src/Invocation.cpp \
|
||||
src/MainWindow.cpp, \
|
||||
|
|
49
src/App.cpp
49
src/App.cpp
|
@ -14,6 +14,7 @@
|
|||
#include "Config.h"
|
||||
#include "Entry.h"
|
||||
#include "Feed.h"
|
||||
#include "FeedController.h"
|
||||
#include "Invocation.h"
|
||||
#include "MainWindow.h"
|
||||
#include "Mimetypes.h"
|
||||
|
@ -31,7 +32,6 @@ main(int argc, char** argv)
|
|||
app->cfg = new Config;
|
||||
app->cfg->Load();
|
||||
|
||||
|
||||
if (argc == 1)
|
||||
app->Run();
|
||||
else
|
||||
|
@ -49,14 +49,55 @@ void
|
|||
cliStart(int argc, char** argv)
|
||||
{
|
||||
invocation(argc, argv);
|
||||
((App*)be_app)->cfg->targetFeeds.DoForEach(&processFeed);
|
||||
BList targetFeeds = ((App*)be_app)->cfg->targetFeeds;
|
||||
|
||||
for (int i = 0; i < targetFeeds.CountItems(); i++) {
|
||||
Feed* newFeed = new Feed(((BString*)targetFeeds.ItemAt(i))->String());
|
||||
BMessage* enqueue = new BMessage(kEnqueueFeed);
|
||||
|
||||
enqueue->AddData("feeds", B_RAW_TYPE, newFeed, sizeof(Feed));
|
||||
|
||||
((App*)be_app)->MessageReceived(enqueue);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
App::App() : BApplication("application/x-vnd.Pogger")
|
||||
{
|
||||
MainWindow* mainWin = new MainWindow();
|
||||
mainWin->Show();
|
||||
fMainWindow = new MainWindow();
|
||||
fFeedController = new FeedController();
|
||||
fMainWindow->Show();
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
App::MessageReceived(BMessage* msg)
|
||||
{
|
||||
switch (msg->what)
|
||||
{
|
||||
case kEnqueueFeed:
|
||||
{
|
||||
fFeedController->MessageReceived(msg);
|
||||
break;
|
||||
}
|
||||
case kClearQueue:
|
||||
{
|
||||
break;
|
||||
}
|
||||
case kQueueProgress:
|
||||
{
|
||||
fMainWindow->MessageReceived(msg);
|
||||
}
|
||||
case kDownloadComplete:
|
||||
{
|
||||
fFeedController->MessageReceived(msg);
|
||||
}
|
||||
default:
|
||||
{
|
||||
// BApplication::MessageReceived(msg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
16
src/App.h
16
src/App.h
|
@ -10,14 +10,23 @@
|
|||
#include <Application.h>
|
||||
|
||||
|
||||
class BMessage;
|
||||
class Config;
|
||||
class FeedController;
|
||||
class MainWindow;
|
||||
|
||||
|
||||
class App : public BApplication
|
||||
{
|
||||
public:
|
||||
App(void);
|
||||
Config* cfg;
|
||||
App(void);
|
||||
void MessageReceived(BMessage* msg);
|
||||
|
||||
Config* cfg;
|
||||
MainWindow* fMainWindow;
|
||||
|
||||
private:
|
||||
FeedController* fFeedController;
|
||||
};
|
||||
|
||||
|
||||
|
@ -25,9 +34,10 @@ int main ( int, char** );
|
|||
void cliStart ( int, char** );
|
||||
|
||||
|
||||
|
||||
extern const char* configPath;
|
||||
extern BString usageMsg;
|
||||
|
||||
|
||||
#endif
|
||||
#endif // APP_H
|
||||
|
||||
|
|
18
src/Feed.cpp
18
src/Feed.cpp
|
@ -26,6 +26,13 @@ Feed::Feed(BString path)
|
|||
}
|
||||
|
||||
|
||||
Feed::Feed(Feed* feed)
|
||||
: Feed()
|
||||
{
|
||||
SetCachePath(feed->GetCachePath());
|
||||
}
|
||||
|
||||
|
||||
Feed::Feed()
|
||||
{
|
||||
title = BString("");
|
||||
|
@ -171,7 +178,7 @@ Feed::SetCachePath (BString givenPath)
|
|||
}
|
||||
|
||||
|
||||
// count the amount of siblings to an element of given type name
|
||||
// Count the amount of siblings to an element of given type name
|
||||
int
|
||||
Feed::xmlCountSiblings (tinyxml2::XMLElement* xsibling, const char* sibling_name)
|
||||
{
|
||||
|
@ -184,7 +191,7 @@ Feed::xmlCountSiblings (tinyxml2::XMLElement* xsibling, const char* sibling_name
|
|||
}
|
||||
|
||||
|
||||
// add the given entry to the feed, if appropriate
|
||||
// Add the given entry to the feed, if appropriate
|
||||
bool
|
||||
Feed::AddEntry (Entry* newEntry)
|
||||
{
|
||||
|
@ -286,6 +293,13 @@ Feed::GetHomeUrl()
|
|||
}
|
||||
|
||||
|
||||
BString
|
||||
Feed::GetXmlUrl()
|
||||
{
|
||||
return xmlUrl;
|
||||
}
|
||||
|
||||
|
||||
// set the update time for a feed
|
||||
bool
|
||||
Feed::SetDate(const char* dateCStr)
|
||||
|
|
|
@ -20,6 +20,7 @@ class BUrl;
|
|||
class Feed {
|
||||
public:
|
||||
Feed(BString);
|
||||
Feed(Feed*);
|
||||
Feed();
|
||||
|
||||
virtual void Parse();
|
||||
|
@ -29,6 +30,8 @@ public:
|
|||
bool IsRss();
|
||||
bool IsAtom();
|
||||
|
||||
BString FetchRemoteFeed();
|
||||
|
||||
bool AddEntry(Entry*);
|
||||
BList GetEntries();
|
||||
bool SetTitle(const char*);
|
||||
|
@ -40,6 +43,7 @@ public:
|
|||
bool SetHomeUrl(const char*);
|
||||
bool SetHomeUrl(tinyxml2::XMLElement*);
|
||||
BString GetHomeUrl();
|
||||
BString GetXmlUrl();
|
||||
bool SetDate(const char*);
|
||||
bool SetDate(tinyxml2::XMLElement*);
|
||||
BDateTime GetDate();
|
||||
|
@ -49,7 +53,6 @@ public:
|
|||
|
||||
protected:
|
||||
void EnsureCached();
|
||||
BString FetchRemoteFeed();
|
||||
int xmlCountSiblings(tinyxml2::XMLElement*, const char*);
|
||||
|
||||
BString title;
|
||||
|
|
|
@ -0,0 +1,135 @@
|
|||
/*
|
||||
* Copyright 2020, Jaidyn Levesque <jadedctrl@teknik.io>
|
||||
* All rights reserved. Distributed under the terms of the MIT license.
|
||||
*/
|
||||
|
||||
#include "FeedController.h"
|
||||
|
||||
#include <Message.h>
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
#include "App.h"
|
||||
#include "AtomFeed.h"
|
||||
#include "Entry.h"
|
||||
#include "RssFeed.h"
|
||||
|
||||
|
||||
FeedController::FeedController()
|
||||
:
|
||||
fDownloadThread(0),
|
||||
fParseThread(0)
|
||||
{
|
||||
fDownloadThread = spawn_thread(_DownloadLoop, "here, eat this",
|
||||
B_NORMAL_PRIORITY, NULL);
|
||||
fParseThread = spawn_thread(_ParseLoop, "oki tnx nomnomnom",
|
||||
B_NORMAL_PRIORITY, NULL);
|
||||
resume_thread(fDownloadThread);
|
||||
resume_thread(fParseThread);
|
||||
}
|
||||
|
||||
|
||||
FeedController::~FeedController()
|
||||
{
|
||||
kill_thread(fDownloadThread);
|
||||
kill_thread(fParseThread);
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
FeedController::MessageReceived(BMessage* msg)
|
||||
{
|
||||
switch (msg->what)
|
||||
{
|
||||
case kEnqueueFeed:
|
||||
{
|
||||
int i = 0;
|
||||
const void* data;
|
||||
ssize_t size = sizeof(Feed);
|
||||
|
||||
while (msg->HasData("feeds", B_RAW_TYPE, i)) {
|
||||
msg->FindData("feeds", B_RAW_TYPE, i, &data, &size);
|
||||
send_data(fDownloadThread, msg->what, data, size);
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case kClearQueue:
|
||||
{
|
||||
break;
|
||||
}
|
||||
case kDownloadComplete:
|
||||
{
|
||||
int i = 0;
|
||||
const void* data;
|
||||
ssize_t size = sizeof(Feed);
|
||||
|
||||
while (msg->HasData("feeds", B_RAW_TYPE, i)) {
|
||||
msg->FindData("feeds", B_RAW_TYPE, i, &data, &size);
|
||||
if (((Feed*)data)->IsUpdated() == true)
|
||||
send_data(fParseThread, msg->what, data, size);
|
||||
i++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
// BWindow::MessageReceived(msg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int32
|
||||
FeedController::_DownloadLoop(void* ignored)
|
||||
{
|
||||
thread_id sender;
|
||||
Feed* feedBuffer = new Feed();
|
||||
|
||||
while (receive_data(&sender, (void*)feedBuffer, sizeof(Feed)) != 0) {
|
||||
feedBuffer->FetchRemoteFeed();
|
||||
|
||||
BMessage* downloaded = new BMessage(kDownloadComplete);
|
||||
downloaded->AddData("feeds", B_RAW_TYPE, feedBuffer, sizeof(Feed));
|
||||
|
||||
|
||||
((App*)be_app)->MessageReceived(downloaded);
|
||||
}
|
||||
|
||||
delete(feedBuffer);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int32
|
||||
FeedController::_ParseLoop(void* ignored)
|
||||
{
|
||||
thread_id sender;
|
||||
Feed* feedBuffer = new Feed();
|
||||
|
||||
while (receive_data(&sender, (void*)feedBuffer, sizeof(Feed)) != 0) {
|
||||
BList entries;
|
||||
|
||||
if (feedBuffer->IsAtom()) {
|
||||
AtomFeed* feed = (AtomFeed*)malloc(sizeof(AtomFeed));
|
||||
feed = new AtomFeed(feedBuffer);
|
||||
feed->Parse();
|
||||
entries = feed->GetEntries();
|
||||
delete(feed);
|
||||
}
|
||||
if (feedBuffer->IsRss()) {
|
||||
RssFeed* feed = (RssFeed*)malloc(sizeof(RssFeed));
|
||||
feed = new RssFeed(feedBuffer);
|
||||
feed->Parse();
|
||||
entries = feed->GetEntries();
|
||||
delete(feed);
|
||||
}
|
||||
|
||||
for (int i = 0; i < entries.CountItems(); i++)
|
||||
((Entry*)entries.ItemAt(i))->Filetize(true);
|
||||
}
|
||||
|
||||
delete (feedBuffer);
|
||||
return 0;
|
||||
}
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Copyright 2020, Jaidyn Levesque <jadedctrl@teknik.io>
|
||||
* All rights reserved. Distributed under the terms of the MIT license.
|
||||
*/
|
||||
#ifndef FEEDCONTROLLER_H
|
||||
#define FEEDCONTROLLER_H
|
||||
|
||||
#include <SupportDefs.h>
|
||||
#include <OS.h>
|
||||
|
||||
class BMessage;
|
||||
|
||||
|
||||
enum
|
||||
{
|
||||
kEnqueueFeed = 'fenq',
|
||||
kClearQueue = 'frmq',
|
||||
kDownloadComplete = 'fdpr',
|
||||
kQueueProgress = 'fqpr'
|
||||
};
|
||||
|
||||
|
||||
class FeedController{
|
||||
public:
|
||||
FeedController();
|
||||
~FeedController();
|
||||
void MessageReceived(BMessage* msg);
|
||||
|
||||
private:
|
||||
static int32 _DownloadLoop(void* ignored);
|
||||
static int32 _ParseLoop(void* ignored);
|
||||
|
||||
thread_id fDownloadThread;
|
||||
thread_id fParseThread;
|
||||
};
|
||||
|
||||
|
||||
#endif // FEEDCONTROLLER_H
|
||||
|
|
@ -124,41 +124,3 @@ freeargInvocation(int argc, char** argv, int optind)
|
|||
}
|
||||
|
||||
|
||||
bool
|
||||
processEntry(void* entry)
|
||||
{
|
||||
Entry* entryPtr = (Entry*)entry;
|
||||
entryPtr->Filetize(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
bool
|
||||
processFeed(void* feedArg)
|
||||
{
|
||||
BString* feedStr = (BString*)feedArg;
|
||||
Feed* testFeed = new Feed(*(feedStr));
|
||||
BList entries;
|
||||
|
||||
if (testFeed->IsUpdated() == false
|
||||
&& ((App*)be_app)->cfg->updateFeeds == true)
|
||||
return false;
|
||||
|
||||
if (testFeed->IsAtom()) {
|
||||
AtomFeed* feed = (AtomFeed*)malloc(sizeof(AtomFeed));
|
||||
feed = new AtomFeed(testFeed);
|
||||
feed->Parse();
|
||||
entries = feed->GetEntries();
|
||||
}
|
||||
if (testFeed->IsRss()) {
|
||||
RssFeed* feed = (RssFeed*)malloc(sizeof(RssFeed));
|
||||
feed = new RssFeed(testFeed);
|
||||
feed->Parse();
|
||||
entries = feed->GetEntries();
|
||||
}
|
||||
|
||||
entries.DoForEach(&processEntry);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
|
|
Ŝarĝante…
Reference in New Issue