Support HTTP(S) feeds (start cache)

This commit is contained in:
Jaidyn Ann 2020-07-13 12:31:52 -05:00
parent 6b7756f50b
commit e69646a42c
11 changed files with 74 additions and 35 deletions

View File

@ -36,7 +36,6 @@ SRCS = \
src/ProtocolListener.cpp, \
src/Config.cpp, \
src/Util.cpp, \
src/parsing.cpp, \
src/Pogger.cpp
# Specify the resource definition files to use. Full or relative paths can be

View File

@ -4,14 +4,14 @@
#include "Util.h"
#include "AtomFeed.h"
AtomFeed::AtomFeed ( BString path, BString outputPath )
AtomFeed::AtomFeed ( BString path, Config* cfg )
{
title = BString("Untitled Feed");
description = BString("");
homeUrl = BString("");
xmlUrl = BString("");
filePath = path;
outputDir = outputPath;
outputDir = cfg->outDir;
}
// ----------------------------------------------------------------------------

View File

@ -11,7 +11,7 @@
class AtomFeed: public Feed {
public:
AtomFeed ( BString, BString );
AtomFeed ( BString, Config* );
void Parse ( Config* );
void RootParse ( Config*, tinyxml2::XMLElement* );

View File

@ -6,16 +6,17 @@ Config::Config () {
verbose = false;
daemon = true;
will_save = false;
updateFeeds = false;
}
// !! handle file status
void
Config::Load ()
{
if ( configPath == NULL )
configPath = BString( "/boot/home/config/settings/Pogger/" );
if ( configDir == NULL )
configDir = BString( "/boot/home/config/settings/Pogger/" );
BString filename = BString(configPath);
BString filename = BString(configDir);
filename.Append("settings");
BFile* file = new BFile( filename.String(), B_READ_ONLY );
status_t result = file->InitCheck();
@ -27,8 +28,8 @@ Config::Load ()
mimetype = BString( storage.GetString("mimetype", "text/xml") );
if ( outDir == NULL)
outDir = BString( storage.GetString("outDir", "/boot/home/feeds/") );
// if ( cacheDir == NULL)
// cacheDir = BString( storage.GetString("cacheDir", "/boot/home/config/cache/Pogger/") );
if ( cacheDir == NULL)
cacheDir = BString( storage.GetString("cacheDir", "/boot/home/config/cache/Pogger/") );
delete file;
}
@ -36,10 +37,10 @@ Config::Load ()
void
Config::Save ()
{
if ( configPath == NULL )
configPath = BString( "/boot/home/config/settings/Pogger/" );
if ( configDir == NULL )
configDir = BString( "/boot/home/config/settings/Pogger/" );
BPath* cfgPath = new BPath( configPath.String(), NULL, true );
BPath* cfgPath = new BPath( configDir.String(), NULL, true );
BEntry* cfgEntry = new BEntry( cfgPath->Path() );
BDirectory* cfgDir = new BDirectory;
@ -49,14 +50,14 @@ Config::Save ()
cfgDir->CreateDirectory( cfgPath->Path(), NULL );
BMessage storage;
BString filename = BString( configPath ).Append("/settings");
BString filename = BString( configDir ).Append("/settings");
BFile* file = new BFile( filename.String(), B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE );
status_t result = file->InitCheck();
storage.AddString( "mimetype", mimetype.String() );
storage.AddString( "outDir", outDir.String() );
// storage.AddString( "cacheDir", cacheDir.String() );
storage.AddString( "cacheDir", cacheDir.String() );
storage.Flatten( file );
}

View File

@ -16,9 +16,12 @@ public:
BDateTime minDate;
BDateTime maxDate;
BString configPath;
BString configDir;
BString cacheDir;
bool will_save;
bool updateFeeds;
Config ( );
void Load ( );

View File

@ -4,13 +4,14 @@
#include "Util.h"
#include "Feed.h"
Feed::Feed ( BString path )
Feed::Feed ( BString path, Config* cfg )
{
title = BString( "Untitled Feed" );
description = BString( "Nondescript, N/A." );
homeUrl = BString("");
xmlUrl = BString("");
filePath = path;
filePath = GetCachePath( path, cfg );
}
Feed::Feed ( ) {
@ -22,6 +23,34 @@ Feed::Feed () {
// ----------------------------------------------------------------------------
BString
Feed::GetCachePath ( BString falsePath, Config* cfg )
{
BUrl falseUrl = BUrl(falsePath);
BString protocol = falseUrl.Protocol().String();
if ( protocol == NULL && falseUrl.UrlString() != NULL )
return falsePath;
if ( protocol != BString("http") && protocol != BString("https") )
return NULL;
BString splitName = falseUrl.Host( );
splitName.Append( falseUrl.Path() );
splitName.ReplaceAll("/", "_");
BString filename = cfg->cacheDir;
filename.Append(splitName);
BFile* cacheFile = new BFile( filename, B_READ_WRITE | B_CREATE_FILE );
if ( cfg->verbose )
printf( "Saving %s to %s...\n", falsePath.String(), filename.String() );
webFetch( falseUrl, cacheFile );
return filename;
}
// ----------------------------------------------------------------------------
bool
Feed::IsRss ( )
{

View File

@ -10,7 +10,7 @@
class Feed {
public:
Feed ( BString );
Feed ( BString, Config* );
Feed ( );
BString title;
@ -37,6 +37,7 @@ public:
bool IsAtom ( );
protected:
BString GetCachePath ( BString, Config* );
int xmlCountSiblings ( tinyxml2::XMLElement*, const char* );
};

View File

@ -45,6 +45,7 @@ invocation ( int argc, char** argv, Config** cfgPtr )
{ "help", no_argument, 0, 'h' },
{ "verbose", no_argument, 0, 'v' },
{ "config", required_argument, 0, 'c' },
{ "cache", required_argument, 0, 'C' },
{ "before", required_argument, 0, 't' },
{ "after", required_argument, 0, 'T' },
{ "output", required_argument, 0, 'O' },
@ -55,7 +56,7 @@ invocation ( int argc, char** argv, Config** cfgPtr )
while (true) {
opterr = 0;
int c = getopt_long(argc, argv, "+hsvDm:O:T:t:c:", sLongOptions, NULL);
int c = getopt_long(argc, argv, "+hsvDm:O:T:t:c:C:", sLongOptions, NULL);
switch (c) {
case -1:
@ -64,7 +65,10 @@ invocation ( int argc, char** argv, Config** cfgPtr )
case 'h':
return usage();
case 'c':
cfg->configPath = BString( optarg );
cfg->configDir = BString( optarg );
break;
case 'C':
cfg->cacheDir = BString( optarg );
break;
case 's':
cfg->will_save = true;
@ -140,18 +144,20 @@ bool
processFeed ( void* feedArg )
{
BString* feedStr = (BString*)feedArg;
Feed* testFeed = new Feed( *(feedStr) );
Feed* testFeed = new Feed( *(feedStr), main_cfg );
BList entries;
if ( testFeed->IsAtom() ) {
printf("Atom\n");
AtomFeed* feed = (AtomFeed*)malloc( sizeof(AtomFeed) );
feed = new AtomFeed( *(feedStr), main_cfg->outDir );
feed = new AtomFeed( testFeed->filePath, main_cfg );
feed->Parse(main_cfg);
entries = feed->entries;
}
if ( testFeed->IsRss() ) {
printf("RSS\n");
RssFeed* feed = (RssFeed*)malloc( sizeof(RssFeed) );
feed = new RssFeed( *(feedStr), main_cfg->outDir );
feed = new RssFeed( testFeed->filePath, main_cfg );
feed->Parse(main_cfg);
entries = feed->entries;
}

View File

@ -7,7 +7,7 @@
class ProtocolListener : public BUrlProtocolListener
{
public:
ProtocolListener(bool traceLogging);
ProtocolListener ( bool );
virtual ~ProtocolListener();
virtual void DataReceived(BUrlRequest*, const char*, off_t, ssize_t);

View File

@ -4,14 +4,14 @@
#include "Util.h"
#include "RssFeed.h"
RssFeed::RssFeed ( BString path, BString outputPath )
RssFeed::RssFeed ( BString path, Config* cfg )
{
title = BString("Untitled Feed");
description = BString("");
homeUrl = BString("");
xmlUrl = BString("");
filePath = path;
outputDir = outputPath;
outputDir = cfg->outDir;
}
// ----------------------------------------------------------------------------

View File

@ -11,7 +11,7 @@
class RssFeed: public Feed {
public:
RssFeed ( BString, BString );
RssFeed ( BString, Config* );
void Parse ( Config* );
void RootParse ( Config*, tinyxml2::XMLElement* );