Multi-threaded parsing/downloading

This commit is contained in:
Jaidyn Ann 2021-01-09 16:53:39 -06:00
parent aa9755b108
commit 0e0bb2ed91
8 changed files with 253 additions and 48 deletions

View File

@ -35,6 +35,7 @@ SRCS = \
src/Entry.cpp, \ src/Entry.cpp, \
src/EntriesView.cpp, \ src/EntriesView.cpp, \
src/Feed.cpp, \ src/Feed.cpp, \
src/FeedController.cpp \
src/FeedsView.cpp, \ src/FeedsView.cpp, \
src/Invocation.cpp \ src/Invocation.cpp \
src/MainWindow.cpp, \ src/MainWindow.cpp, \

View File

@ -14,6 +14,7 @@
#include "Config.h" #include "Config.h"
#include "Entry.h" #include "Entry.h"
#include "Feed.h" #include "Feed.h"
#include "FeedController.h"
#include "Invocation.h" #include "Invocation.h"
#include "MainWindow.h" #include "MainWindow.h"
#include "Mimetypes.h" #include "Mimetypes.h"
@ -31,7 +32,6 @@ main(int argc, char** argv)
app->cfg = new Config; app->cfg = new Config;
app->cfg->Load(); app->cfg->Load();
if (argc == 1) if (argc == 1)
app->Run(); app->Run();
else else
@ -49,14 +49,55 @@ void
cliStart(int argc, char** argv) cliStart(int argc, char** argv)
{ {
invocation(argc, argv); invocation(argc, argv);
((App*)be_app)->cfg->targetFeeds.DoForEach(&processFeed); BList targetFeeds = ((App*)be_app)->cfg->targetFeeds;
for (int i = 0; i < targetFeeds.CountItems(); i++) {
Feed* newFeed = new Feed(((BString*)targetFeeds.ItemAt(i))->String());
BMessage* enqueue = new BMessage(kEnqueueFeed);
enqueue->AddData("feeds", B_RAW_TYPE, newFeed, sizeof(Feed));
((App*)be_app)->MessageReceived(enqueue);
}
} }
App::App() : BApplication("application/x-vnd.Pogger") App::App() : BApplication("application/x-vnd.Pogger")
{ {
MainWindow* mainWin = new MainWindow(); fMainWindow = new MainWindow();
mainWin->Show(); fFeedController = new FeedController();
fMainWindow->Show();
}
void
App::MessageReceived(BMessage* msg)
{
switch (msg->what)
{
case kEnqueueFeed:
{
fFeedController->MessageReceived(msg);
break;
}
case kClearQueue:
{
break;
}
case kQueueProgress:
{
fMainWindow->MessageReceived(msg);
}
case kDownloadComplete:
{
fFeedController->MessageReceived(msg);
}
default:
{
// BApplication::MessageReceived(msg);
break;
}
}
} }

View File

@ -10,14 +10,23 @@
#include <Application.h> #include <Application.h>
class BMessage;
class Config; class Config;
class FeedController;
class MainWindow;
class App : public BApplication class App : public BApplication
{ {
public: public:
App(void); App(void);
void MessageReceived(BMessage* msg);
Config* cfg; Config* cfg;
MainWindow* fMainWindow;
private:
FeedController* fFeedController;
}; };
@ -25,9 +34,10 @@ int main ( int, char** );
void cliStart ( int, char** ); void cliStart ( int, char** );
extern const char* configPath; extern const char* configPath;
extern BString usageMsg; extern BString usageMsg;
#endif #endif // APP_H

View File

@ -26,6 +26,13 @@ Feed::Feed(BString path)
} }
Feed::Feed(Feed* feed)
: Feed()
{
SetCachePath(feed->GetCachePath());
}
Feed::Feed() Feed::Feed()
{ {
title = BString(""); title = BString("");
@ -171,7 +178,7 @@ Feed::SetCachePath (BString givenPath)
} }
// count the amount of siblings to an element of given type name // Count the amount of siblings to an element of given type name
int int
Feed::xmlCountSiblings (tinyxml2::XMLElement* xsibling, const char* sibling_name) Feed::xmlCountSiblings (tinyxml2::XMLElement* xsibling, const char* sibling_name)
{ {
@ -184,7 +191,7 @@ Feed::xmlCountSiblings (tinyxml2::XMLElement* xsibling, const char* sibling_name
} }
// add the given entry to the feed, if appropriate // Add the given entry to the feed, if appropriate
bool bool
Feed::AddEntry (Entry* newEntry) Feed::AddEntry (Entry* newEntry)
{ {
@ -286,6 +293,13 @@ Feed::GetHomeUrl()
} }
BString
Feed::GetXmlUrl()
{
return xmlUrl;
}
// set the update time for a feed // set the update time for a feed
bool bool
Feed::SetDate(const char* dateCStr) Feed::SetDate(const char* dateCStr)

View File

@ -20,6 +20,7 @@ class BUrl;
class Feed { class Feed {
public: public:
Feed(BString); Feed(BString);
Feed(Feed*);
Feed(); Feed();
virtual void Parse(); virtual void Parse();
@ -29,6 +30,8 @@ public:
bool IsRss(); bool IsRss();
bool IsAtom(); bool IsAtom();
BString FetchRemoteFeed();
bool AddEntry(Entry*); bool AddEntry(Entry*);
BList GetEntries(); BList GetEntries();
bool SetTitle(const char*); bool SetTitle(const char*);
@ -40,6 +43,7 @@ public:
bool SetHomeUrl(const char*); bool SetHomeUrl(const char*);
bool SetHomeUrl(tinyxml2::XMLElement*); bool SetHomeUrl(tinyxml2::XMLElement*);
BString GetHomeUrl(); BString GetHomeUrl();
BString GetXmlUrl();
bool SetDate(const char*); bool SetDate(const char*);
bool SetDate(tinyxml2::XMLElement*); bool SetDate(tinyxml2::XMLElement*);
BDateTime GetDate(); BDateTime GetDate();
@ -49,7 +53,6 @@ public:
protected: protected:
void EnsureCached(); void EnsureCached();
BString FetchRemoteFeed();
int xmlCountSiblings(tinyxml2::XMLElement*, const char*); int xmlCountSiblings(tinyxml2::XMLElement*, const char*);
BString title; BString title;

135
src/FeedController.cpp Normal file
View File

@ -0,0 +1,135 @@
/*
* Copyright 2020, Jaidyn Levesque <jadedctrl@teknik.io>
* All rights reserved. Distributed under the terms of the MIT license.
*/
#include "FeedController.h"
#include <Message.h>
#include <cstdio>
#include "App.h"
#include "AtomFeed.h"
#include "Entry.h"
#include "RssFeed.h"
FeedController::FeedController()
:
fDownloadThread(0),
fParseThread(0)
{
fDownloadThread = spawn_thread(_DownloadLoop, "here, eat this",
B_NORMAL_PRIORITY, NULL);
fParseThread = spawn_thread(_ParseLoop, "oki tnx nomnomnom",
B_NORMAL_PRIORITY, NULL);
resume_thread(fDownloadThread);
resume_thread(fParseThread);
}
FeedController::~FeedController()
{
kill_thread(fDownloadThread);
kill_thread(fParseThread);
}
void
FeedController::MessageReceived(BMessage* msg)
{
switch (msg->what)
{
case kEnqueueFeed:
{
int i = 0;
const void* data;
ssize_t size = sizeof(Feed);
while (msg->HasData("feeds", B_RAW_TYPE, i)) {
msg->FindData("feeds", B_RAW_TYPE, i, &data, &size);
send_data(fDownloadThread, msg->what, data, size);
i++;
}
break;
}
case kClearQueue:
{
break;
}
case kDownloadComplete:
{
int i = 0;
const void* data;
ssize_t size = sizeof(Feed);
while (msg->HasData("feeds", B_RAW_TYPE, i)) {
msg->FindData("feeds", B_RAW_TYPE, i, &data, &size);
if (((Feed*)data)->IsUpdated() == true)
send_data(fParseThread, msg->what, data, size);
i++;
}
break;
}
default:
{
// BWindow::MessageReceived(msg);
break;
}
}
}
int32
FeedController::_DownloadLoop(void* ignored)
{
thread_id sender;
Feed* feedBuffer = new Feed();
while (receive_data(&sender, (void*)feedBuffer, sizeof(Feed)) != 0) {
feedBuffer->FetchRemoteFeed();
BMessage* downloaded = new BMessage(kDownloadComplete);
downloaded->AddData("feeds", B_RAW_TYPE, feedBuffer, sizeof(Feed));
((App*)be_app)->MessageReceived(downloaded);
}
delete(feedBuffer);
return 0;
}
int32
FeedController::_ParseLoop(void* ignored)
{
thread_id sender;
Feed* feedBuffer = new Feed();
while (receive_data(&sender, (void*)feedBuffer, sizeof(Feed)) != 0) {
BList entries;
if (feedBuffer->IsAtom()) {
AtomFeed* feed = (AtomFeed*)malloc(sizeof(AtomFeed));
feed = new AtomFeed(feedBuffer);
feed->Parse();
entries = feed->GetEntries();
delete(feed);
}
if (feedBuffer->IsRss()) {
RssFeed* feed = (RssFeed*)malloc(sizeof(RssFeed));
feed = new RssFeed(feedBuffer);
feed->Parse();
entries = feed->GetEntries();
delete(feed);
}
for (int i = 0; i < entries.CountItems(); i++)
((Entry*)entries.ItemAt(i))->Filetize(true);
}
delete (feedBuffer);
return 0;
}

39
src/FeedController.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright 2020, Jaidyn Levesque <jadedctrl@teknik.io>
* All rights reserved. Distributed under the terms of the MIT license.
*/
#ifndef FEEDCONTROLLER_H
#define FEEDCONTROLLER_H
#include <SupportDefs.h>
#include <OS.h>
class BMessage;
enum
{
kEnqueueFeed = 'fenq',
kClearQueue = 'frmq',
kDownloadComplete = 'fdpr',
kQueueProgress = 'fqpr'
};
class FeedController{
public:
FeedController();
~FeedController();
void MessageReceived(BMessage* msg);
private:
static int32 _DownloadLoop(void* ignored);
static int32 _ParseLoop(void* ignored);
thread_id fDownloadThread;
thread_id fParseThread;
};
#endif // FEEDCONTROLLER_H

View File

@ -124,41 +124,3 @@ freeargInvocation(int argc, char** argv, int optind)
} }
bool
processEntry(void* entry)
{
Entry* entryPtr = (Entry*)entry;
entryPtr->Filetize(false);
return false;
}
bool
processFeed(void* feedArg)
{
BString* feedStr = (BString*)feedArg;
Feed* testFeed = new Feed(*(feedStr));
BList entries;
if (testFeed->IsUpdated() == false
&& ((App*)be_app)->cfg->updateFeeds == true)
return false;
if (testFeed->IsAtom()) {
AtomFeed* feed = (AtomFeed*)malloc(sizeof(AtomFeed));
feed = new AtomFeed(testFeed);
feed->Parse();
entries = feed->GetEntries();
}
if (testFeed->IsRss()) {
RssFeed* feed = (RssFeed*)malloc(sizeof(RssFeed));
feed = new RssFeed(testFeed);
feed->Parse();
entries = feed->GetEntries();
}
entries.DoForEach(&processEntry);
return false;
}