llhttpassetstorage.cpp

Go to the documentation of this file.

#include "linden_common.h"

#include "llhttpassetstorage.h"

#include <sys/stat.h>

#include "indra_constants.h"
#include "message.h"
#include "llvfile.h"
#include "llvfs.h"

#ifdef LL_STANDALONE
# include <zlib.h>
#else
# include "zlib/zlib.h"
#endif

const U32 MAX_RUNNING_REQUESTS = 1;
const F32 MAX_PROCESSING_TIME = 0.005f;
const S32 CURL_XFER_BUFFER_SIZE = 65536;
// Try for 30 minutes for now.
const F32 GET_URL_TO_FILE_TIMEOUT = 1800.0f;

const S32 COMPRESSED_INPUT_BUFFER_SIZE = 4096;

const S32 HTTP_OK = 200;
const S32 HTTP_PUT_OK = 201;
const S32 HTTP_NO_CONTENT = 204;
const S32 HTTP_MISSING = 404;
const S32 HTTP_SERVER_BAD_GATEWAY = 502;
const S32 HTTP_SERVER_TEMP_UNAVAILABLE = 503;

// LLTempAssetData
// An asset not stored on central asset store, but on a simulator node somewhere.
struct LLTempAssetData
{
        LLUUID  mAssetID;
        LLUUID  mAgentID;
        std::string     mHostName;
};

// LLHTTPAssetRequest

class LLHTTPAssetRequest : public LLAssetRequest
{
public:
        LLHTTPAssetRequest(LLHTTPAssetStorage *asp, const LLUUID &uuid, 
                                        LLAssetType::EType type, LLAssetStorage::ERequestType rt,
                                        const char *url, CURLM *curl_multi);
        virtual ~LLHTTPAssetRequest();
        
        void setupCurlHandle();
        void cleanupCurlHandle();

        void    prepareCompressedUpload();
        void    finishCompressedUpload();
        size_t  readCompressedData(void* data, size_t size);

        static size_t curlCompressedUploadCallback(
                                        void *data, size_t size, size_t nmemb, void *user_data);

        virtual LLSD getTerseDetails() const;
        virtual LLSD getFullDetails() const;

public:
        LLHTTPAssetStorage *mAssetStoragep;

        CURL  *mCurlHandle;
        CURLM *mCurlMultiHandle;
        char  *mURLBuffer;
        struct curl_slist *mHTTPHeaders;
        LLVFile *mVFile;
        LLUUID  mTmpUUID;
        LLAssetStorage::ERequestType mRequestType;

        bool            mZInitialized;
        z_stream        mZStream;
        char*           mZInputBuffer;
        bool            mZInputExhausted;

        FILE *mFP;
};


LLHTTPAssetRequest::LLHTTPAssetRequest(LLHTTPAssetStorage *asp, 
                                                const LLUUID &uuid, 
                                                LLAssetType::EType type, 
                                                LLAssetStorage::ERequestType rt,
                                                const char *url, 
                                                CURLM *curl_multi)
        : LLAssetRequest(uuid, type),
        mZInitialized(false)
{
        mAssetStoragep = asp;
        mCurlHandle = NULL;
        mCurlMultiHandle = curl_multi;
        mVFile = NULL;
        mRequestType = rt;
        mHTTPHeaders = NULL;
        mFP = NULL;
        mZInputBuffer = NULL;
        mZInputExhausted = false;
        
        mURLBuffer = new char[strlen(url) + 1]; /*Flawfinder: ignore*/
        if (mURLBuffer)
        {
            strcpy(mURLBuffer, url);    /*Flawfinder: ignore*/
        }
}

LLHTTPAssetRequest::~LLHTTPAssetRequest()
{
        // Cleanup/cancel the request
        if (mCurlHandle)
        {
                curl_multi_remove_handle(mCurlMultiHandle, mCurlHandle);
                cleanupCurlHandle();
        }
        if (mHTTPHeaders)
        {
                curl_slist_free_all(mHTTPHeaders);
        }
        delete[] mURLBuffer;
        delete   mVFile;
        finishCompressedUpload();
}

// virtual
LLSD LLHTTPAssetRequest::getTerseDetails() const
{
        LLSD sd = LLAssetRequest::getTerseDetails();

        sd["url"] = mURLBuffer;

        return sd;
}

// virtual
LLSD LLHTTPAssetRequest::getFullDetails() const
{
        LLSD sd = LLAssetRequest::getFullDetails();

        if (mCurlHandle)
        {
                long curl_response = -1;
                long curl_connect = -1;
                double curl_total_time = -1.0f;
                double curl_size_upload = -1.0f;
                double curl_size_download = -1.0f;
                long curl_content_length_upload = -1;
                long curl_content_length_download = -1;
                long curl_request_size = -1;
                const char* curl_content_type = NULL;

                curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CODE, &curl_response);
                curl_easy_getinfo(mCurlHandle, CURLINFO_HTTP_CONNECTCODE, &curl_connect);
                curl_easy_getinfo(mCurlHandle, CURLINFO_TOTAL_TIME, &curl_total_time);
                curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_UPLOAD,  &curl_size_upload);
                curl_easy_getinfo(mCurlHandle, CURLINFO_SIZE_DOWNLOAD, &curl_size_download);
                curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_UPLOAD,   &curl_content_length_upload);
                curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &curl_content_length_download);
                curl_easy_getinfo(mCurlHandle, CURLINFO_REQUEST_SIZE, &curl_request_size);
                curl_easy_getinfo(mCurlHandle, CURLINFO_CONTENT_TYPE, &curl_content_type);

                sd["curl_response_code"] = (int) curl_response;
                sd["curl_http_connect_code"] = (int) curl_connect;
                sd["curl_total_time"] = curl_total_time;
                sd["curl_size_upload"]   = curl_size_upload;
                sd["curl_size_download"] = curl_size_download;
                sd["curl_content_length_upload"]   = (int) curl_content_length_upload;
                sd["curl_content_length_download"] = (int) curl_content_length_download;
                sd["curl_request_size"] = (int) curl_request_size;
                if (curl_content_type)
                {
                        sd["curl_content_type"] = curl_content_type;
                }
                else
                {
                        sd["curl_content_type"] = "";
                }
        }

        sd["temp_id"] = mTmpUUID;
        sd["request_type"] = LLAssetStorage::getRequestName(mRequestType);
        sd["z_initialized"] = mZInitialized;
        sd["z_input_exhausted"] = mZInputExhausted;

        S32 file_size = -1;
        if (mFP)
        {
                struct stat file_stat;
                int file_desc = fileno(mFP);
                if ( fstat(file_desc, &file_stat) == 0)
                {
                        file_size = file_stat.st_size;
                }
        }
        sd["file_size"] = file_size;

        return sd;
}


void LLHTTPAssetRequest::setupCurlHandle()
{
        mCurlHandle = curl_easy_init();
        curl_easy_setopt(mCurlHandle, CURLOPT_NOSIGNAL, 1);
        curl_easy_setopt(mCurlHandle, CURLOPT_NOPROGRESS, 1);
        curl_easy_setopt(mCurlHandle, CURLOPT_URL, mURLBuffer);
        curl_easy_setopt(mCurlHandle, CURLOPT_PRIVATE, this);
        if (LLAssetStorage::RT_DOWNLOAD == mRequestType)
        {
                curl_easy_setopt(mCurlHandle, CURLOPT_ENCODING, "");
                // only do this on downloads, as uploads 
                // to some apache configs (like our test grids)
                // mistakenly claim the response is gzip'd if the resource
                // name ends in .gz, even though in a PUT, the response is
                // just plain HTML saying "created"
        }
        /* Remove the Pragma: no-cache header that libcurl inserts by default;
           we want the cached version, if possible. */
        if (mZInitialized)
        {
                curl_easy_setopt(mCurlHandle, CURLOPT_PROXY, "");
                        // disable use of proxy, which can't handle chunked transfers
        }
        mHTTPHeaders = curl_slist_append(mHTTPHeaders, "Pragma:");
        // resist the temptation to explicitly add the Transfer-Encoding: chunked
        // header here - invokes a libCURL bug
        curl_easy_setopt(mCurlHandle, CURLOPT_HTTPHEADER, mHTTPHeaders);
        if (mAssetStoragep)
        {
                // Set the appropriate pending upload or download flag
                mAssetStoragep->addRunningRequest(mRequestType, this);
        }
        else
        {
                llerrs << "LLHTTPAssetRequest::setupCurlHandle - No asset storage associated with this request!" << llendl;
        }
}

void LLHTTPAssetRequest::cleanupCurlHandle()
{
        curl_easy_cleanup(mCurlHandle);
        if (mAssetStoragep)
        {
                // Terminating a request.  Thus upload or download is no longer pending.
                mAssetStoragep->removeRunningRequest(mRequestType, this);
        }
        else
        {
                llerrs << "LLHTTPAssetRequest::~LLHTTPAssetRequest - No asset storage associated with this request!" << llendl;
        }
        mCurlHandle = NULL;
}

void LLHTTPAssetRequest::prepareCompressedUpload()
{
        mZStream.next_in = Z_NULL;
        mZStream.avail_in = 0;
        mZStream.zalloc = Z_NULL;
        mZStream.zfree = Z_NULL;
        mZStream.opaque = Z_NULL;

        int r = deflateInit2(&mZStream,
                        1,                      // compression level
                        Z_DEFLATED,     // the only method defined
                        15 + 16,        // the default windowBits + gzip header flag
                        8,                      // the default memLevel
                        Z_DEFAULT_STRATEGY);

        if (r != Z_OK)
        {
                llerrs << "LLHTTPAssetRequest::prepareCompressedUpload defalateInit2() failed" << llendl;
        }

        mZInitialized = true;
        mZInputBuffer = new char[COMPRESSED_INPUT_BUFFER_SIZE];
        mZInputExhausted = false;

        mVFile = new LLVFile(gAssetStorage->mVFS,
                                        getUUID(), getType(), LLVFile::READ);
}

void LLHTTPAssetRequest::finishCompressedUpload()
{
        if (mZInitialized)
        {
                llinfos << "LLHTTPAssetRequest::finishCompressedUpload: "
                        << "read " << mZStream.total_in << " byte asset file, "
                        << "uploaded " << mZStream.total_out << " byte compressed asset"
                        << llendl;

                deflateEnd(&mZStream);
                delete[] mZInputBuffer;
        }
}

size_t LLHTTPAssetRequest::readCompressedData(void* data, size_t size)
{
        mZStream.next_out = (Bytef*)data;
        mZStream.avail_out = size;

        while (mZStream.avail_out > 0)
        {
                if (mZStream.avail_in == 0 && !mZInputExhausted)
                {
                        S32 to_read = llmin(COMPRESSED_INPUT_BUFFER_SIZE,
                                                        (S32)(mVFile->getSize() - mVFile->tell()));
                        
                        if ( to_read > 0 )
                        {
                                mVFile->read((U8*)mZInputBuffer, to_read); /*Flawfinder: ignore*/
                                mZStream.next_in = (Bytef*)mZInputBuffer;
                                mZStream.avail_in = mVFile->getLastBytesRead();
                        }

                        mZInputExhausted = mZStream.avail_in == 0;
                }

                int r = deflate(&mZStream,
                                        mZInputExhausted ? Z_FINISH : Z_NO_FLUSH);

                if (r == Z_STREAM_END || r < 0 || mZInputExhausted)
                {
                        if (r < 0)
                        {
                                llwarns << "LLHTTPAssetRequest::readCompressedData: deflate returned error code " 
                                                << (S32) r << llendl;
                        }
                        break;
                }
        }

        return size - mZStream.avail_out;
}

//static
size_t LLHTTPAssetRequest::curlCompressedUploadCallback(
                void *data, size_t size, size_t nmemb, void *user_data)
{
        size_t num_read = 0;

        if (gAssetStorage)
        {
                CURL *curl_handle = (CURL *)user_data;
                LLHTTPAssetRequest *req = NULL;
                curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);
                if (req)
                {
                        num_read = req->readCompressedData(data, size * nmemb);
                }
        }

        return num_read;
}

// LLHTTPAssetStorage


LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer,
                                                                         LLVFS *vfs, const LLHost &upstream_host,
                                                                         const char *web_host,
                                                                         const char *local_web_host,
                                                                         const char *host_name)
        : LLAssetStorage(msg, xfer, vfs, upstream_host)
{
        _init(web_host, local_web_host, host_name);
}

LLHTTPAssetStorage::LLHTTPAssetStorage(LLMessageSystem *msg, LLXferManager *xfer,
                                                                           LLVFS *vfs,
                                                                           const char *web_host,
                                                                           const char *local_web_host,
                                                                           const char *host_name)
        : LLAssetStorage(msg, xfer, vfs)
{
        _init(web_host, local_web_host, host_name);
}

void LLHTTPAssetStorage::_init(const char *web_host, const char *local_web_host, const char* host_name)
{
        mBaseURL = web_host;
        mLocalBaseURL = local_web_host;
        mHostName = host_name;

        // Do not change this "unless you are familiar with and mean to control 
        // internal operations of libcurl"
        // - http://curl.haxx.se/libcurl/c/curl_global_init.html
        curl_global_init(CURL_GLOBAL_ALL);

        mCurlMultiHandle = curl_multi_init();
}

LLHTTPAssetStorage::~LLHTTPAssetStorage()
{
        curl_multi_cleanup(mCurlMultiHandle);
        mCurlMultiHandle = NULL;
        
        curl_global_cleanup();
}

// storing data is simpler than getting it, so we just overload the whole method
void LLHTTPAssetStorage::storeAssetData(
        const LLUUID& uuid,
        LLAssetType::EType type,
        LLAssetStorage::LLStoreAssetCallback callback,
        void* user_data,
        bool temp_file,
        bool is_priority,
        bool store_local,
        const LLUUID& requesting_agent_id,
        bool user_waiting,
        F64 timeout)
{
        if (mVFS->getExists(uuid, type))
        {
                LLAssetRequest *req = new LLAssetRequest(uuid, type);
                req->mUpCallback    = callback;
                req->mUserData      = user_data;
                req->mRequestingAgentID = requesting_agent_id;
                req->mIsUserWaiting = user_waiting;
                req->mTimeout       = timeout;

                // this will get picked up and transmitted in checkForTimeouts
                if(store_local)
                {
                        mPendingLocalUploads.push_back(req);
                }
                else if(is_priority)
                {
                        mPendingUploads.push_front(req);
                }
                else
                {
                        mPendingUploads.push_back(req);
                }
        }
        else
        {
                llwarns << "AssetStorage: attempt to upload non-existent vfile " << uuid << ":" << LLAssetType::lookup(type) << llendl;
                if (callback)
                {
                        callback(uuid, user_data,  LL_ERR_ASSET_REQUEST_NONEXISTENT_FILE, LL_EXSTAT_NONEXISTENT_FILE);
                }
        }
}

// virtual
void LLHTTPAssetStorage::storeAssetData(
        const char* filename,
        const LLUUID& asset_id,
        LLAssetType::EType asset_type,
        LLStoreAssetCallback callback,
        void* user_data,
        bool temp_file,
        bool is_priority,
        bool user_waiting,
        F64 timeout)
{
        llinfos << "LLAssetStorage::storeAssetData (legacy)" << asset_id << ":" << LLAssetType::lookup(asset_type) << llendl;

        LLLegacyAssetRequest *legacy = new LLLegacyAssetRequest;

        legacy->mUpCallback = callback;
        legacy->mUserData = user_data;

        FILE *fp = LLFile::fopen(filename, "rb"); /*Flawfinder: ignore*/
        if (fp)
        {
                LLVFile file(mVFS, asset_id, asset_type, LLVFile::WRITE);

                fseek(fp, 0, SEEK_END);
                S32 size = ftell(fp);
                fseek(fp, 0, SEEK_SET);

                file.setMaxSize(size);

                const S32 buf_size = 65536;
                U8 copy_buf[buf_size];
                while ((size = (S32)fread(copy_buf, 1, buf_size, fp)))
                {
                        file.write(copy_buf, size);
                }
                fclose(fp);

                // if this upload fails, the caller needs to setup a new tempfile for us
                if (temp_file)
                {
                        LLFile::remove(filename);
                }
                
                storeAssetData(
                        asset_id,
                        asset_type,
                        legacyStoreDataCallback,
                        (void**)legacy,
                        temp_file,
                        is_priority,
                        false,
                        LLUUID::null,
                        user_waiting,
                        timeout);
        }
        else
        {
                if (callback)
                {
                        callback(LLUUID::null, user_data, LL_ERR_CANNOT_OPEN_FILE, LL_EXSTAT_BLOCKED_FILE);
                }
                delete legacy;
        }
}

// virtual
LLSD LLHTTPAssetStorage::getPendingDetails(LLAssetStorage::ERequestType rt,
                                                                                LLAssetType::EType asset_type,
                                                                                const std::string& detail_prefix) const
{
        LLSD sd = LLAssetStorage::getPendingDetails(rt, asset_type, detail_prefix);
        const request_list_t* running = getRunningList(rt);
        if (running)
        {
                // Loop through the pending requests sd, and add extra info about its running status.
                S32 num_pending = sd["requests"].size();
                S32 i;
                for (i = 0; i < num_pending; ++i)
                {
                        LLSD& pending = sd["requests"][i];
                        // See if this pending request is running.
                        const LLAssetRequest* req = findRequest(running, 
                                                                        LLAssetType::lookup(pending["type"].asString().c_str()),
                                                                        pending["asset_id"]);
                        if (req)
                        {
                                // Keep the detail_url so we don't have to rebuild it.
                                LLURI detail_url = pending["detail"];
                                pending = req->getTerseDetails();
                                pending["detail"] = detail_url;
                                pending["is_running"] = true;
                        }
                        else
                        {
                                pending["is_running"] = false;
                        }
                }
        }
        return sd;
}

// virtual
LLSD LLHTTPAssetStorage::getPendingRequest(LLAssetStorage::ERequestType rt,
                                                                                LLAssetType::EType asset_type,
                                                                                const LLUUID& asset_id) const
{
        // Look for this asset in the running list first.
        const request_list_t* running = getRunningList(rt);
        if (running)
        {
                LLSD sd = LLAssetStorage::getPendingRequest(running, asset_type, asset_id);
                if (sd)
                {
                        sd["is_running"] = true;
                        return sd;
                }
        }
        LLSD sd = LLAssetStorage::getPendingRequest(rt, asset_type, asset_id);
        if (sd)
        {
                sd["is_running"] = false;
        }
        return sd;
}

// virtual
bool LLHTTPAssetStorage::deletePendingRequest(LLAssetStorage::ERequestType rt,
                                                                                        LLAssetType::EType asset_type,
                                                                                        const LLUUID& asset_id)
{
        // Try removing this from the running list first.
        request_list_t* running = getRunningList(rt);
        if (running)
        {
                LLAssetRequest* req = findRequest(running, asset_type, asset_id);
                if (req)
                {
                        // Remove this request from the running list to get it out of curl.
                        running->remove(req);
                        
                        // Find this request in the pending list, so we can move it to the end of the line.
                        request_list_t* pending = getRequestList(rt);
                        if (pending)
                        {
                                request_list_t::iterator result = std::find_if(pending->begin(), pending->end(),
                                                                                std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req));
                                if (pending->end() != result)
                                {
                                        // This request was found in the pending list.  Move it to the end!
                                        LLAssetRequest* pending_req = *result;
                                        pending->remove(pending_req);

                                        if (!pending_req->mIsUserWaiting)                               //A user is waiting on this request.  Toss it.
                                        {
                                                pending->push_back(pending_req);
                                        }
                                        else
                                        {
                                                if (pending_req->mUpCallback)   //Clean up here rather than _callUploadCallbacks because this request is already cleared the req.
                                                {
                                                        pending_req->mUpCallback(pending_req->getUUID(), pending_req->mUserData, -1, LL_EXSTAT_REQUEST_DROPPED);
                                                }

                                        }

                                        llinfos << "Asset " << getRequestName(rt) << " request for "
                                                        << asset_id << "." << LLAssetType::lookup(asset_type)
                                                        << " removed from curl and placed at the end of the pending queue."
                                                        << llendl;
                                }
                                else
                                {
                                        llwarns << "Unable to find pending " << getRequestName(rt) << " request for "
                                                        << asset_id << "." << LLAssetType::lookup(asset_type) << llendl;
                                }
                        }
                        delete req;

                        return true;
                }
        }
        return LLAssetStorage::deletePendingRequest(rt, asset_type, asset_id);
}

// internal requester, used by getAssetData in superclass
void LLHTTPAssetStorage::_queueDataRequest(const LLUUID& uuid, LLAssetType::EType type,
                                                                                  void (*callback)(LLVFS *vfs, const LLUUID&, LLAssetType::EType, void *, S32, LLExtStat),
                                                                                  void *user_data, BOOL duplicate,
                                                                                   BOOL is_priority)
{
        // stash the callback info so we can find it after we get the response message
        LLAssetRequest *req = new LLAssetRequest(uuid, type);
        req->mDownCallback = callback;
        req->mUserData = user_data;
        req->mIsPriority = is_priority;

        // this will get picked up and downloaded in checkForTimeouts

        //
        // HAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACK!  Asset requests were taking too long and timing out.
        // Since texture requests are the LEAST sensitive (on the simulator) to being delayed, add
        // non-texture requests to the front, and add texture requests to the back.  The theory is
        // that we always want them first, even if they're out of order.
        //
        
        if (req->getType() == LLAssetType::AT_TEXTURE)
        {
                mPendingDownloads.push_back(req);
        }
        else
        {
                mPendingDownloads.push_front(req);
        }
}

LLAssetRequest* LLHTTPAssetStorage::findNextRequest(LLAssetStorage::request_list_t& pending, 
                                                                                                        LLAssetStorage::request_list_t& running)
{
        // Early exit if the running list is full, or we don't have more pending than running.
        if (running.size() >= MAX_RUNNING_REQUESTS
                || pending.size() <= running.size()) return NULL;

        // Look for the first pending request that is not already running.
        request_list_t::iterator running_begin = running.begin();
        request_list_t::iterator running_end   = running.end();

        request_list_t::iterator pending_iter = pending.begin();
        request_list_t::iterator pending_end  = pending.end();
        // Loop over all pending requests until we miss finding it in the running list.
        for (; pending_iter != pending.end(); ++pending_iter)
        {
                LLAssetRequest* req = *pending_iter;
                // Look for this pending request in the running list.
                if (running_end == std::find_if(running_begin, running_end, 
                                                std::bind2nd(ll_asset_request_equal<LLAssetRequest*>(), req)))
                {
                        // It isn't running!  Return it.
                        return req;
                }
        }
        return NULL;
}

// overloaded to additionally move data to/from the webserver
void LLHTTPAssetStorage::checkForTimeouts()
{
        CURLMcode mcode;
        LLAssetRequest *req;
        while ( (req = findNextRequest(mPendingDownloads, mRunningDownloads)) )
        {
                // Setup this curl download request
                // We need to generate a new request here
                // since the one in the list could go away
                char tmp_url[MAX_STRING]; /*Flawfinder: ignore*/
                char uuid_str[UUID_STR_LENGTH]; /*Flawfinder: ignore*/
                req->getUUID().toString(uuid_str);
                std::string base_url = getBaseURL(req->getUUID(), req->getType());
                snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", base_url.c_str() , uuid_str, LLAssetType::lookup(req->getType()));     /* Flawfinder: ignore */

                LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), 
                                                                                req->getType(), RT_DOWNLOAD, tmp_url, mCurlMultiHandle);
                new_req->mTmpUUID.generate();

                // Sets pending download flag internally
                new_req->setupCurlHandle();
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE);
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &curlDownCallback);
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEDATA, new_req->mCurlHandle);
        
                mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
                if (mcode > CURLM_OK)
                {
                        // Failure.  Deleting the pending request will remove it from the running
                        // queue, and push it to the end of the pending queue.
                        new_req->cleanupCurlHandle();
                        deletePendingRequest(RT_DOWNLOAD, new_req->getType(), new_req->getUUID());
                        break;
                }
                else
                {
                        llinfos << "Requesting " << new_req->mURLBuffer << llendl;
                }
        }

        while ( (req = findNextRequest(mPendingUploads, mRunningUploads)) )
        {
                // setup this curl upload request

                bool do_compress = req->getType() == LLAssetType::AT_OBJECT;

                char tmp_url[MAX_STRING];/*Flawfinder: ignore*/
                char uuid_str[UUID_STR_LENGTH];/*Flawfinder: ignore*/
                req->getUUID().toString(uuid_str);
                snprintf(tmp_url, sizeof(tmp_url),                                      /* Flawfinder: ignore */
                                do_compress ? "%s/%s.%s.gz" : "%s/%s.%s",
                                mBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType())); 

                LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), 
                                                                        req->getType(), RT_UPLOAD, tmp_url, mCurlMultiHandle);

                if (req->mIsUserWaiting) //If a user is waiting on a realtime response, we want to perserve information across upload attempts.
                {
                        new_req->mTime          = req->mTime;
                        new_req->mTimeout       = req->mTimeout;
                        new_req->mIsUserWaiting = req->mIsUserWaiting;
                }

                if (do_compress)
                {
                        new_req->prepareCompressedUpload();
                }

                // Sets pending upload flag internally
                new_req->setupCurlHandle();
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_UPLOAD, 1);
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback);

                if (do_compress)
                {
                        curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION,
                                        &LLHTTPAssetRequest::curlCompressedUploadCallback);
                }
                else
                {
                        LLVFile file(mVFS, req->getUUID(), req->getType());
                        curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize());
                        curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION,
                                        &curlUpCallback);
                }
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle);
        
                mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
                if (mcode > CURLM_OK)
                {
                        // Failure.  Deleting the pending request will remove it from the running
                        // queue, and push it to the end of the pending queue.
                        new_req->cleanupCurlHandle();
                        deletePendingRequest(RT_UPLOAD, new_req->getType(), new_req->getUUID());
                        break;
                }
                else
                {
                        llinfos << "Requesting PUT " << new_req->mURLBuffer << llendl;
                }
                // Pending upload will have been flagged by the request
        }

        while ( (req = findNextRequest(mPendingLocalUploads, mRunningLocalUploads)) )
        {
                // setup this curl upload request
                LLVFile file(mVFS, req->getUUID(), req->getType());

                char tmp_url[MAX_STRING]; /*Flawfinder: ignore*/
                char uuid_str[UUID_STR_LENGTH]; /*Flawfinder: ignore*/
                req->getUUID().toString(uuid_str);
                
                // KLW - All temporary uploads are saved locally "http://localhost:12041/asset"
                snprintf(tmp_url, sizeof(tmp_url), "%s/%36s.%s", mLocalBaseURL.c_str(), uuid_str, LLAssetType::lookup(req->getType()));         /* Flawfinder: ignore */

                LLHTTPAssetRequest *new_req = new LLHTTPAssetRequest(this, req->getUUID(), 
                                                                                req->getType(), RT_LOCALUPLOAD, tmp_url, mCurlMultiHandle);
                new_req->mRequestingAgentID = req->mRequestingAgentID;

                // Sets pending upload flag internally
                new_req->setupCurlHandle();
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_PUT, 1);
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_INFILESIZE, file.getSize());
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_WRITEFUNCTION, &nullOutputCallback);
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READFUNCTION, &curlUpCallback);
                curl_easy_setopt(new_req->mCurlHandle, CURLOPT_READDATA, new_req->mCurlHandle);
        
                mcode = curl_multi_add_handle(mCurlMultiHandle, new_req->mCurlHandle);
                if (mcode > CURLM_OK)
                {
                        // Failure.  Deleting the pending request will remove it from the running
                        // queue, and push it to the end of the pending queue.
                        new_req->cleanupCurlHandle();
                        deletePendingRequest(RT_LOCALUPLOAD, new_req->getType(), new_req->getUUID());
                        break;
                }
                else
                {
                        llinfos << "TAT: LLHTTPAssetStorage::checkForTimeouts() : pending local!"
                                << " Requesting PUT " << new_req->mURLBuffer << llendl;
                }
                // Pending upload will have been flagged by the request
        }
        S32 count = 0;
        int queue_length;
        do
        {
                mcode = curl_multi_perform(mCurlMultiHandle, &queue_length);
                count++;
        } while (mcode == CURLM_CALL_MULTI_PERFORM && (count < 5));

        CURLMsg *curl_msg;
        do
        {
                curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length);
                if (curl_msg && curl_msg->msg == CURLMSG_DONE)
                {
                        long curl_result = 0;
                        S32 xfer_result = LL_ERR_NOERR;

                        LLHTTPAssetRequest *req = NULL;
                        curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &req);
                                                                
                        // TODO: Throw curl_result at all callbacks.
                        curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result);
                        if (RT_UPLOAD == req->mRequestType || RT_LOCALUPLOAD == req->mRequestType)
                        {
                                if (curl_msg->data.result == CURLE_OK && 
                                        (   curl_result == HTTP_OK 
                                         || curl_result == HTTP_PUT_OK 
                                         || curl_result == HTTP_NO_CONTENT))
                                {
                                        llinfos << "Success uploading " << req->getUUID() << " to " << req->mURLBuffer << llendl;
                                        if (RT_LOCALUPLOAD == req->mRequestType)
                                        {
                                                addTempAssetData(req->getUUID(), req->mRequestingAgentID, mHostName);
                                        }
                                }
                                else if (curl_msg->data.result == CURLE_COULDNT_CONNECT ||
                                                curl_msg->data.result == CURLE_OPERATION_TIMEOUTED ||
                                                curl_result == HTTP_SERVER_BAD_GATEWAY ||
                                                curl_result == HTTP_SERVER_TEMP_UNAVAILABLE)
                                {
                                        llwarns << "Re-requesting upload for " << req->getUUID() << ".  Received upload error to " << req->mURLBuffer <<
                                                " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;

                                        //if (req->mIsUserWaiting)
                                        //{
                                        //      deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID());
                                        //}
                                }
                                else
                                {
                                        llwarns << "Failure uploading " << req->getUUID() << " to " << req->mURLBuffer <<
                                                " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;

                                        xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
                                }

                                if (!(curl_msg->data.result == CURLE_COULDNT_CONNECT ||
                                                curl_msg->data.result == CURLE_OPERATION_TIMEOUTED ||
                                                curl_result == HTTP_SERVER_BAD_GATEWAY ||
                                                curl_result == HTTP_SERVER_TEMP_UNAVAILABLE))
                                {
                                        // shared upload finished callback
                                        // in the base class, this is called from processUploadComplete
                                        _callUploadCallbacks(req->getUUID(), req->getType(), (xfer_result == 0), LL_EXSTAT_CURL_RESULT | curl_result);
                                        // Pending upload flag will get cleared when the request is deleted
                                }
                        }
                        else if (RT_DOWNLOAD == req->mRequestType)
                        {
                                if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK)
                                {
                                        if (req->mVFile && req->mVFile->getSize() > 0)
                                        {                                       
                                                llinfos << "Success downloading " << req->mURLBuffer << ", size " << req->mVFile->getSize() << llendl;

                                                req->mVFile->rename(req->getUUID(), req->getType());
                                        }
                                        else
                                        {
                                                // *TODO: if this actually indicates a bad asset on the server
                                                // (not certain at this point), then delete it
                                                llwarns << "Found " << req->mURLBuffer << " to be zero size" << llendl;
                                                xfer_result = LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE;
                                        }
                                }
                                else
                                {
                                        // KLW - TAT See if an avatar owns this texture, and if so request re-upload.
                                        llwarns << "Failure downloading " << req->mURLBuffer << 
                                                " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;

                                        xfer_result = (curl_result == HTTP_MISSING) ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED;

                                        if (req->mVFile)
                                        {
                                                req->mVFile->remove();
                                        }
                                }

                                // call the static callback for transfer completion
                                // this will cleanup all requests for this asset, including ours
                                downloadCompleteCallback(
                                        xfer_result,
                                        req->getUUID(),
                                        req->getType(),
                                        (void *)req,
                                        LL_EXSTAT_CURL_RESULT | curl_result);
                                // Pending download flag will get cleared when the request is deleted
                        }
                        else
                        {
                                // nothing, just axe this request
                                // currently this can only mean an asset delete
                        }

                        // Deleting clears the pending upload/download flag if it's set and the request is transferring
                        delete req;
                        req = NULL;
                }

        } while (curl_msg && queue_length > 0);
        

        // Cleanup 
        // We want to bump to the back of the line any running uploads that have timed out.
        bumpTimedOutUploads();

        LLAssetStorage::checkForTimeouts();
}

void LLHTTPAssetStorage::bumpTimedOutUploads()
{
        bool user_waiting=FALSE;

        F64 mt_secs = LLMessageSystem::getMessageTimeSeconds();

        if (mPendingUploads.size())
        {
                request_list_t::iterator it = mPendingUploads.begin();
                LLAssetRequest* req = *it;
                user_waiting=req->mIsUserWaiting;
        }

        // No point bumping currently running uploads if there are no others in line.
        if (!(mPendingUploads.size() > mRunningUploads.size()) && !user_waiting) 
        {
                return;
        }

        // deletePendingRequest will modify the mRunningUploads list so we don't want to iterate over it.
        request_list_t temp_running = mRunningUploads;

        request_list_t::iterator it = temp_running.begin();
        request_list_t::iterator end = temp_running.end();
        for ( ; it != end; ++it)
        {
                //request_list_t::iterator curiter = iter++;
                LLAssetRequest* req = *it;

                if ( req->mTimeout < (mt_secs - req->mTime) )
                {
                        llwarns << "Asset upload request timed out for "
                                        << req->getUUID() << "."
                                        << LLAssetType::lookup(req->getType()) 
                                        << ", bumping to the back of the line!" << llendl;

                        deletePendingRequest(RT_UPLOAD, req->getType(), req->getUUID());
                }
        }
}

// static
size_t LLHTTPAssetStorage::curlDownCallback(void *data, size_t size, size_t nmemb, void *user_data)
{
        if (!gAssetStorage)
        {
                llwarns << "Missing gAssetStorage, aborting curl download callback!" << llendl;
                return 0;
        }
        S32 bytes = (S32)(size * nmemb);
        CURL *curl_handle = (CURL *)user_data;
        LLHTTPAssetRequest *req = NULL;
        curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);

        if (! req->mVFile)
        {
                req->mVFile = new LLVFile(gAssetStorage->mVFS, req->mTmpUUID, LLAssetType::AT_NONE, LLVFile::APPEND);
        }

        double content_length = 0.0;
        curl_easy_getinfo(curl_handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &content_length);

        // sanitize content_length, reconcile w/ actual data
        S32 file_length = llmax(0, (S32)llmin(content_length, 20000000.0), bytes + req->mVFile->getSize());

        req->mVFile->setMaxSize(file_length);
        req->mVFile->write((U8*)data, bytes);

        return nmemb;
}

// static 
size_t LLHTTPAssetStorage::curlUpCallback(void *data, size_t size, size_t nmemb, void *user_data)
{
        if (!gAssetStorage)
        {
                llwarns << "Missing gAssetStorage, aborting curl download callback!" << llendl;
                return 0;
        }
        CURL *curl_handle = (CURL *)user_data;
        LLHTTPAssetRequest *req = NULL;
        curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);

        if (! req->mVFile)
        {
                req->mVFile = new LLVFile(gAssetStorage->mVFS, req->getUUID(), req->getType(), LLVFile::READ);
        }

        S32 bytes = llmin((S32)(size * nmemb), (S32)(req->mVFile->getSize() - req->mVFile->tell()));

        req->mVFile->read((U8*)data, bytes);/*Flawfinder: ignore*/

        return req->mVFile->getLastBytesRead();
}

// static
size_t LLHTTPAssetStorage::nullOutputCallback(void *data, size_t size, size_t nmemb, void *user_data)
{
        // do nothing, this is here to soak up script output so it doesn't end up on stdout

        return nmemb;
}



// blocking asset fetch which bypasses the VFS
// this is a very limited function for use by the simstate loader and other one-offs
S32 LLHTTPAssetStorage::getURLToFile(const LLUUID& uuid, LLAssetType::EType asset_type, const LLString &url, const char *filename, progress_callback callback, void *userdata)
{
        // *NOTE: There is no guarantee that the uuid and the asset_type match
        // - not that it matters. - Doug
        lldebugs << "LLHTTPAssetStorage::getURLToFile() - " << url << llendl;

        FILE *fp = LLFile::fopen(filename, "wb"); /*Flawfinder: ignore*/
        if (! fp)
        {
                llwarns << "Failed to open " << filename << " for writing" << llendl;
                return LL_ERR_ASSET_REQUEST_FAILED;
        }

        // make sure we use the normal curl setup, even though we don't really need a request object
        LLHTTPAssetRequest req(this, uuid, asset_type, RT_DOWNLOAD, url.c_str(), mCurlMultiHandle);
        req.mFP = fp;
        
        req.setupCurlHandle();
        curl_easy_setopt(req.mCurlHandle, CURLOPT_FOLLOWLOCATION, TRUE);
        curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEFUNCTION, &curlFileDownCallback);
        curl_easy_setopt(req.mCurlHandle, CURLOPT_WRITEDATA, req.mCurlHandle);

        curl_multi_add_handle(mCurlMultiHandle, req.mCurlHandle);
        llinfos << "Requesting as file " << req.mURLBuffer << llendl;

        // braindead curl loop
        int queue_length;
        CURLMsg *curl_msg;
        LLTimer timeout;
        timeout.setTimerExpirySec(GET_URL_TO_FILE_TIMEOUT);
        bool success = false;
        S32 xfer_result = 0;
        do
        {
                curl_multi_perform(mCurlMultiHandle, &queue_length);
                curl_msg = curl_multi_info_read(mCurlMultiHandle, &queue_length);

                if (callback)
                {
                        callback(userdata);
                }

                if ( curl_msg && (CURLMSG_DONE == curl_msg->msg) )
                {
                        success = true;
                }
                else if (timeout.hasExpired())
                {
                        llwarns << "Request for " << url << " has timed out." << llendl;
                        success = false;
                        xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
                        break;
                }
        } while (!success);

        if (success)
        {
                long curl_result = 0;
                curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_HTTP_CODE, &curl_result);
                
                if (curl_result == HTTP_OK && curl_msg->data.result == CURLE_OK)
                {
                        S32 size = ftell(req.mFP);
                        if (size > 0)
                        {
                                // everything seems to be in order
                                llinfos << "Success downloading " << req.mURLBuffer << " to file, size " << size << llendl;
                        }
                        else
                        {
                                llwarns << "Found " << req.mURLBuffer << " to be zero size" << llendl;
                                xfer_result = LL_ERR_ASSET_REQUEST_FAILED;
                        }
                }
                else
                {
                        xfer_result = curl_result == HTTP_MISSING ? LL_ERR_ASSET_REQUEST_NOT_IN_DATABASE : LL_ERR_ASSET_REQUEST_FAILED;
                        llinfos << "Failure downloading " << req.mURLBuffer << 
                                " with result " << curl_easy_strerror(curl_msg->data.result) << ", http result " << curl_result << llendl;
                }
        }

        fclose(fp);
        if (xfer_result)
        {
                LLFile::remove(filename);
        }
        return xfer_result;
}


// static
size_t LLHTTPAssetStorage::curlFileDownCallback(void *data, size_t size, size_t nmemb, void *user_data)
{       
        CURL *curl_handle = (CURL *)user_data;
        LLHTTPAssetRequest *req = NULL;
        curl_easy_getinfo(curl_handle, CURLINFO_PRIVATE, &req);

        if (! req->mFP)
        {
                llwarns << "Missing mFP, aborting curl file download callback!" << llendl;
                return 0;
        }

        return fwrite(data, size, nmemb, req->mFP);
}

LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt)
{
        switch (rt)
        {
        case RT_DOWNLOAD:
                return &mRunningDownloads;
        case RT_UPLOAD:
                return &mRunningUploads;
        case RT_LOCALUPLOAD:
                return &mRunningLocalUploads;
        default:
                return NULL;
        }
}

const LLAssetStorage::request_list_t* LLHTTPAssetStorage::getRunningList(LLAssetStorage::ERequestType rt) const
{
        switch (rt)
        {
        case RT_DOWNLOAD:
                return &mRunningDownloads;
        case RT_UPLOAD:
                return &mRunningUploads;
        case RT_LOCALUPLOAD:
                return &mRunningLocalUploads;
        default:
                return NULL;
        }
}


void LLHTTPAssetStorage::addRunningRequest(ERequestType rt, LLHTTPAssetRequest* request)
{
        request_list_t* requests = getRunningList(rt);
        if (requests)
        {
                requests->push_back(request);
        }
        else
        {
                llerrs << "LLHTTPAssetStorage::addRunningRequest - Request is not an upload OR download, this is bad!" << llendl;
        }
}

void LLHTTPAssetStorage::removeRunningRequest(ERequestType rt, LLHTTPAssetRequest* request)
{
        request_list_t* requests = getRunningList(rt);
        if (requests)
        {
                requests->remove(request);
        }
        else
        {
                llerrs << "LLHTTPAssetStorage::removeRunningRequest - Destroyed request is not an upload OR download, this is bad!" << llendl;
        }
}

// virtual 
void LLHTTPAssetStorage::addTempAssetData(const LLUUID& asset_id, const LLUUID& agent_id, const std::string& host_name)
{
        if (agent_id.isNull() || asset_id.isNull())
        {
                llwarns << "TAT: addTempAssetData bad id's asset_id: " << asset_id << "  agent_id: " << agent_id << llendl;
                return;
        }

        LLTempAssetData temp_asset_data;
        temp_asset_data.mAssetID = asset_id;
        temp_asset_data.mAgentID = agent_id;
        temp_asset_data.mHostName = host_name;

        mTempAssets[asset_id] = temp_asset_data;
}

// virtual
BOOL LLHTTPAssetStorage::hasTempAssetData(const LLUUID& texture_id) const
{
        uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
        BOOL found = (citer != mTempAssets.end());
        return found;
}

// virtual
std::string LLHTTPAssetStorage::getTempAssetHostName(const LLUUID& texture_id) const
{
        uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
        if (citer != mTempAssets.end())
        {
                return citer->second.mHostName;
        }
        else
        {
                return std::string();
        }
}

// virtual 
LLUUID LLHTTPAssetStorage::getTempAssetAgentID(const LLUUID& texture_id) const
{
        uuid_tempdata_map::const_iterator citer = mTempAssets.find(texture_id);
        if (citer != mTempAssets.end())
        {
                return citer->second.mAgentID;
        }
        else
        {
                return LLUUID::null;
        }
}

// virtual 
void LLHTTPAssetStorage::removeTempAssetData(const LLUUID& asset_id)
{
        mTempAssets.erase(asset_id);
}

// virtual 
void LLHTTPAssetStorage::removeTempAssetDataByAgentID(const LLUUID& agent_id)
{
        uuid_tempdata_map::iterator it = mTempAssets.begin();
        uuid_tempdata_map::iterator end = mTempAssets.end();

        while (it != end)
        {
                const LLTempAssetData& asset_data = it->second;
                if (asset_data.mAgentID == agent_id)
                {
                        mTempAssets.erase(it++);
                }
                else
                {
                        ++it;
                }
        }
}

std::string LLHTTPAssetStorage::getBaseURL(const LLUUID& asset_id, LLAssetType::EType asset_type)
{
        if (LLAssetType::AT_TEXTURE == asset_type)
        {
                uuid_tempdata_map::const_iterator citer = mTempAssets.find(asset_id);
                if (citer != mTempAssets.end())
                {
                        const std::string& host_name = citer->second.mHostName;
                        std::string url = llformat(LOCAL_ASSET_URL_FORMAT, host_name.c_str());
                        return url;
                }
        }

        return mBaseURL;
}

void LLHTTPAssetStorage::dumpTempAssetData(const LLUUID& avatar_id) const
{
        uuid_tempdata_map::const_iterator it = mTempAssets.begin();
        uuid_tempdata_map::const_iterator end = mTempAssets.end();
        S32 count = 0;
        for ( ; it != end; ++it)
        {
                const LLTempAssetData& temp_asset_data = it->second;
                if (avatar_id.isNull()
                        || avatar_id == temp_asset_data.mAgentID)
                {
                        llinfos << "TAT: dump agent " << temp_asset_data.mAgentID
                                << " texture " << temp_asset_data.mAssetID
                                << " host " << temp_asset_data.mHostName
                                << llendl;
                        count++;
                }
        }

        if (avatar_id.isNull())
        {
                llinfos << "TAT: dumped " << count << " entries for all avatars" << llendl;
        }
        else
        {
                llinfos << "TAT: dumped " << count << " entries for avatar " << avatar_id << llendl;
        }
}

void LLHTTPAssetStorage::clearTempAssetData()
{
        llinfos << "TAT: Clearing temp asset data map" << llendl;
        mTempAssets.clear();
}

---